1   //========================================================================
2   //Copyright 2004-2008 Mort Bay Consulting Pty. Ltd.
3   //------------------------------------------------------------------------
4   //Licensed under the Apache License, Version 2.0 (the "License");
5   //you may not use this file except in compliance with the License.
6   //You may obtain a copy of the License at 
7   //http://www.apache.org/licenses/LICENSE-2.0
8   //Unless required by applicable law or agreed to in writing, software
9   //distributed under the License is distributed on an "AS IS" BASIS,
10  //WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11  //See the License for the specific language governing permissions and
12  //limitations under the License.
13  //========================================================================
14  
15  package org.mortbay.io.nio;
16  
17  import java.io.IOException;
18  import java.nio.channels.CancelledKeyException;
19  import java.nio.channels.SelectionKey;
20  import java.nio.channels.Selector;
21  import java.nio.channels.ServerSocketChannel;
22  import java.nio.channels.SocketChannel;
23  import java.util.ArrayList;
24  import java.util.Iterator;
25  import java.util.List;
26  
27  import org.mortbay.component.AbstractLifeCycle;
28  import org.mortbay.io.Connection;
29  import org.mortbay.io.EndPoint;
30  import org.mortbay.log.Log;
31  import org.mortbay.thread.Timeout;
32  
33  
34  /* ------------------------------------------------------------ */
35  /**
36   * The Selector Manager manages and number of SelectSets to allow
37   * NIO scheduling to scale to large numbers of connections.
38   * 
39   * @author gregw
40   *
41   */
42  public abstract class SelectorManager extends AbstractLifeCycle
43  {
44      private long _maxIdleTime;
45      private long _lowResourcesConnections;
46      private long _lowResourcesMaxIdleTime;
47      private transient SelectSet[] _selectSet;
48      private int _selectSets=1;
49      private volatile int _set;
50      
51  
52      /* ------------------------------------------------------------ */
53      /**
54       * @param maxIdleTime The maximum period in milli seconds that a connection may be idle before it is closed.
55       * @see {@link #setLowResourcesMaxIdleTime(long)}
56       */
57      public void setMaxIdleTime(long maxIdleTime)
58      {
59          _maxIdleTime=maxIdleTime;
60      }
61      
62      /* ------------------------------------------------------------ */
63      /**
64       * @param selectSets
65       */
66      public void setSelectSets(int selectSets)
67      {
68          long lrc = _lowResourcesConnections * _selectSets; 
69          _selectSets=selectSets;
70          _lowResourcesConnections=lrc/_selectSets;
71      }
72      
73      /* ------------------------------------------------------------ */
74      /**
75       * @return
76       */
77      public long getMaxIdleTime()
78      {
79          return _maxIdleTime;
80      }
81      
82      /* ------------------------------------------------------------ */
83      /**
84       * @return
85       */
86      public int getSelectSets()
87      {
88          return _selectSets;
89      }
90      
91      /* ------------------------------------------------------------ */
92      /** Register a channel
93       * @param channel
94       * @param att Attached Object
95       * @throws IOException
96       */
97      public void register(SocketChannel channel, Object att) throws IOException
98      {
99          int s=_set++; 
100         s=s%_selectSets;
101         SelectSet[] sets=_selectSet;
102         if (sets!=null)
103         {
104             SelectSet set=sets[s];
105             set.addChange(channel,att);
106             set.wakeup();
107         }
108     }
109     
110     /* ------------------------------------------------------------ */
111     /** Register a serverchannel
112      * @param acceptChannel
113      * @return
114      * @throws IOException
115      */
116     public void register(ServerSocketChannel acceptChannel) throws IOException
117     {
118         int s=_set++; 
119         s=s%_selectSets;
120         SelectSet set=_selectSet[s];
121         set.addChange(acceptChannel);
122         set.wakeup();
123     }
124 
125     /* ------------------------------------------------------------ */
126     /**
127      * @return the lowResourcesConnections
128      */
129     public long getLowResourcesConnections()
130     {
131         return _lowResourcesConnections*_selectSets;
132     }
133 
134     /* ------------------------------------------------------------ */
135     /**
136      * Set the number of connections, which if exceeded places this manager in low resources state.
137      * This is not an exact measure as the connection count is averaged over the select sets.
138      * @param lowResourcesConnections the number of connections
139      * @see {@link #setLowResourcesMaxIdleTime(long)}
140      */
141     public void setLowResourcesConnections(long lowResourcesConnections)
142     {
143         _lowResourcesConnections=(lowResourcesConnections+_selectSets-1)/_selectSets;
144     }
145 
146     /* ------------------------------------------------------------ */
147     /**
148      * @return the lowResourcesMaxIdleTime
149      */
150     public long getLowResourcesMaxIdleTime()
151     {
152         return _lowResourcesMaxIdleTime;
153     }
154 
155     /* ------------------------------------------------------------ */
156     /**
157      * @param lowResourcesMaxIdleTime the period in ms that a connection is allowed to be idle when this SelectSet has more connections than {@link #getLowResourcesConnections()}
158      * @see {@link #setMaxIdleTime(long)}
159      */
160     public void setLowResourcesMaxIdleTime(long lowResourcesMaxIdleTime)
161     {
162         _lowResourcesMaxIdleTime=lowResourcesMaxIdleTime;
163     }
164     
165     /* ------------------------------------------------------------ */
166     /**
167      * @param acceptorID
168      * @throws IOException
169      */
170     public void doSelect(int acceptorID) throws IOException
171     {
172         SelectSet[] sets= _selectSet;
173         if (sets!=null && sets.length>acceptorID && sets[acceptorID]!=null)
174             sets[acceptorID].doSelect();
175     }
176 
177     /* ------------------------------------------------------------ */
178     /**
179      * @param key
180      * @return
181      * @throws IOException 
182      */
183     protected abstract SocketChannel acceptChannel(SelectionKey key) throws IOException;
184 
185     /* ------------------------------------------------------------------------------- */
186     public abstract boolean dispatch(Runnable task);
187 
188     /* ------------------------------------------------------------ */
189     /* (non-Javadoc)
190      * @see org.mortbay.component.AbstractLifeCycle#doStart()
191      */
192     protected void doStart() throws Exception
193     {
194         _selectSet = new SelectSet[_selectSets];
195         for (int i=0;i<_selectSet.length;i++)
196             _selectSet[i]= new SelectSet(i);
197 
198         super.doStart();
199     }
200 
201 
202     /* ------------------------------------------------------------------------------- */
203     protected void doStop() throws Exception
204     {
205         SelectSet[] sets= _selectSet;
206         _selectSet=null;
207         if (sets!=null)
208             for (int i=0;i<sets.length;i++)
209             {
210                 SelectSet set = sets[i];
211                 if (set!=null)
212                     set.stop();
213             }
214         super.doStop();
215     }
216 
217     /* ------------------------------------------------------------ */
218     /**
219      * @param endpoint
220      */
221     protected abstract void endPointClosed(SelectChannelEndPoint endpoint);
222 
223     /* ------------------------------------------------------------ */
224     /**
225      * @param endpoint
226      */
227     protected abstract void endPointOpened(SelectChannelEndPoint endpoint);
228 
229     /* ------------------------------------------------------------------------------- */
230     protected abstract Connection newConnection(SocketChannel channel, SelectChannelEndPoint endpoint);
231 
232     /* ------------------------------------------------------------ */
233     /**
234      * @param channel
235      * @param selectSet
236      * @param sKey
237      * @return
238      * @throws IOException
239      */
240     protected abstract SelectChannelEndPoint newEndPoint(SocketChannel channel, SelectorManager.SelectSet selectSet, SelectionKey sKey) throws IOException;
241 
242     /* ------------------------------------------------------------------------------- */
243     protected void connectionFailed(SocketChannel channel,Throwable ex,Object attachment)
244     {
245         Log.warn(ex);
246     }
247     
248     /* ------------------------------------------------------------------------------- */
249     /* ------------------------------------------------------------------------------- */
250     /* ------------------------------------------------------------------------------- */
251     public class SelectSet 
252     {
253         private transient int _change;
254         private transient List<Object>[] _changes;
255         private transient Timeout _idleTimeout;
256         private transient int _nextSet;
257         private transient Timeout _timeout;
258         private transient Selector _selector;
259         private transient int _setID;
260         private transient boolean _selecting;
261         private transient int _jvmBug;
262         
263         /* ------------------------------------------------------------ */
264         SelectSet(int acceptorID) throws Exception
265         {
266             _setID=acceptorID;
267 
268             _idleTimeout = new Timeout(this);
269             _idleTimeout.setDuration(getMaxIdleTime());
270             _timeout = new Timeout(this);
271             _timeout.setDuration(0L);
272 
273             // create a selector;
274             _selector = Selector.open();
275             _changes = new List[] {new ArrayList(),new ArrayList()};
276             _change=0;
277         }
278         
279         /* ------------------------------------------------------------ */
280         public void addChange(Object point)
281         {
282             synchronized (_changes)
283             {
284                 _changes[_change].add(point);
285                 if (point instanceof SocketChannel)
286                     _changes[_change].add(null);
287             }
288         }
289         
290         /* ------------------------------------------------------------ */
291         public void addChange(SocketChannel channel, Object att)
292         {   
293             synchronized (_changes)
294             {
295                 _changes[_change].add(channel);
296                 _changes[_change].add(att);
297             }
298         }
299         
300         /* ------------------------------------------------------------ */
301         public void cancelIdle(Timeout.Task task)
302         {
303             task.cancel();
304         }
305 
306         /* ------------------------------------------------------------ */
307         /**
308          * Select and dispatch tasks found from changes and the selector.
309          * 
310          * @throws IOException
311          */
312         public void doSelect() throws IOException
313         {
314             try
315             {
316                 List<?> changes;
317                 synchronized (_changes)
318                 {
319                     changes=_changes[_change];
320                     _change=_change==0?1:0;
321                     _selecting=true;
322                 }
323 
324                 // Make any key changes required
325                 for (int i = 0; i < changes.size(); i++)
326                 {
327                     try
328                     {
329                         Object o = changes.get(i);
330                         if (o instanceof EndPoint)
331                         {
332                             // Update the operations for a key.
333                             SelectChannelEndPoint endpoint = (SelectChannelEndPoint)o;
334                             endpoint.doUpdateKey();
335                         }
336                         else if (o instanceof Runnable)
337                         {
338                             dispatch((Runnable)o);
339                         }
340                         else if (o instanceof SocketChannel)
341                         {
342                             // finish accepting/connecting this connection
343                             SocketChannel channel=(SocketChannel)o;
344                             Object att = changes.get(++i);
345 
346                             if (channel.isConnected())
347                             {
348                                 SelectionKey key = channel.register(_selector,SelectionKey.OP_READ,att);
349                                 SelectChannelEndPoint endpoint = newEndPoint(channel,this,key);
350                                 key.attach(endpoint);
351                                 endpoint.schedule();
352                             }
353                             else
354                             {
355                                 channel.register(_selector,SelectionKey.OP_CONNECT,att);
356                             }
357 
358                         }
359                         else if (o instanceof ServerSocketChannel)
360                         {
361                             ServerSocketChannel channel = (ServerSocketChannel)o;
362                             channel.register(getSelector(),SelectionKey.OP_ACCEPT);
363                         }
364                         else
365                             throw new IllegalArgumentException(o.toString());
366                     }
367                     catch (CancelledKeyException e)
368                     {
369                         if (isRunning())
370                             Log.warn(e);
371                         else
372                             Log.debug(e);
373                     }
374                 }
375                 changes.clear();
376 
377                 long idle_next = 0;
378                 long retry_next = 0;
379                 long now=System.currentTimeMillis();
380                 synchronized (this)
381                 {
382                     _idleTimeout.setNow(now);
383                     _timeout.setNow(now);
384                     if (_lowResourcesConnections>0 && _selector.keys().size()>_lowResourcesConnections)
385                         _idleTimeout.setDuration(_lowResourcesMaxIdleTime);
386                     else 
387                         _idleTimeout.setDuration(_maxIdleTime);
388                     idle_next=_idleTimeout.getTimeToNext();
389                     retry_next=_timeout.getTimeToNext();
390                 }
391 
392                 // workout how low to wait in select
393                 long wait = 1000L;  // not getMaxIdleTime() as the now value of the idle timers needs to be updated.
394                 if (idle_next >= 0 && wait > idle_next)
395                     wait = idle_next;
396                 if (wait > 0 && retry_next >= 0 && wait > retry_next)
397                     wait = retry_next;
398     
399                 // Do the select.
400                 if (wait > 0) 
401                 {
402                     long before=now;
403                     int selected=_selector.select(wait);
404                     now = System.currentTimeMillis();
405                     _idleTimeout.setNow(now);
406                     _timeout.setNow(now);
407 
408                     // Look for JVM bug  http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6403933
409                     if (selected==0  && (now-before)<wait/2)
410                     {
411                         if (_jvmBug++>5) 
412                         {
413                             // Probably JVM BUG!    
414                             for (SelectionKey key: _selector.keys())
415                             {    
416                                 if (key.interestOps()==0 && key.isValid())
417                                     key.cancel();
418                             }
419                             _selector.selectNow();
420                         } 
421                     }
422                     else
423                         _jvmBug=0;
424                 }
425                 else 
426                 {
427                     _selector.selectNow();
428                     _jvmBug=0;
429                 }
430 
431                 // have we been destroyed while sleeping\
432                 if (_selector==null || !_selector.isOpen())
433                     return;
434 
435                 // Look for things to do
436                 for (SelectionKey key: _selector.selectedKeys())
437                 {   
438                     try
439                     {
440                         if (!key.isValid())
441                         {
442                             key.cancel();
443                             SelectChannelEndPoint endpoint = (SelectChannelEndPoint)key.attachment();
444                             if (endpoint != null)
445                                 endpoint.doUpdateKey();
446                             continue;
447                         }
448 
449                         Object att = key.attachment();
450                         if (att instanceof SelectChannelEndPoint)
451                         {
452                             ((SelectChannelEndPoint)att).schedule();
453                         }
454                         else if (key.isAcceptable())
455                         {
456                             SocketChannel channel = acceptChannel(key);
457                             if (channel==null)
458                                 continue;
459 
460                             channel.configureBlocking(false);
461 
462                             // TODO make it reluctant to leave 0
463                             _nextSet=++_nextSet%_selectSet.length;
464 
465                             // Is this for this selectset
466                             if (_nextSet==_setID)
467                             {
468                                 // bind connections to this select set.
469                                 SelectionKey cKey = channel.register(_selectSet[_nextSet].getSelector(), SelectionKey.OP_READ);
470                                 SelectChannelEndPoint endpoint=newEndPoint(channel,_selectSet[_nextSet],cKey);
471                                 cKey.attach(endpoint);
472                                 if (endpoint != null)
473                                     endpoint.schedule();
474                             }
475                             else
476                             {
477                                 // nope - give it to another.
478                                 _selectSet[_nextSet].addChange(channel);
479                                 _selectSet[_nextSet].wakeup();
480                             }
481                         }
482                         else if (key.isConnectable())
483                         {
484                             // Complete a connection of a registered channel
485                             SocketChannel channel = (SocketChannel)key.channel();
486                             boolean connected=false;
487                             try
488                             {
489                                 connected=channel.finishConnect();
490                             }
491                             catch(Exception e)
492                             {
493                                 connectionFailed(channel,e,att);
494                             }
495                             finally
496                             {
497                                 if (connected)
498                                 {
499                                     key.interestOps(SelectionKey.OP_READ);
500                                     SelectChannelEndPoint endpoint = newEndPoint(channel,this,key);
501                                     key.attach(endpoint);
502                                     endpoint.schedule();
503                                 }
504                                 else
505                                 {
506                                     key.cancel();
507                                 }
508                             }
509                         }
510                         else
511                         {
512                             // Wrap readable registered channel in an endpoint
513                             SocketChannel channel = (SocketChannel)key.channel();
514                             SelectChannelEndPoint endpoint = newEndPoint(channel,this,key);
515                             key.attach(endpoint);
516                             if (key.isReadable())
517                                 endpoint.schedule();                           
518                         }
519                         key = null;
520                     }
521                     catch (CancelledKeyException e)
522                     {
523                         Log.ignore(e);
524                     }
525                     catch (Exception e)
526                     {
527                         if (isRunning())
528                             Log.warn(e);
529                         else
530                             Log.ignore(e);
531 
532                         if (key != null && !(key.channel() instanceof ServerSocketChannel) && key.isValid())
533                             key.cancel();
534                     }
535                 }
536                 
537                 // Everything always handled
538                 _selector.selectedKeys().clear();
539 
540                 // tick over the timers
541                 _idleTimeout.tick(now);
542                 _timeout.tick(now);
543             }
544             catch (CancelledKeyException e)
545             {
546                 Log.ignore(e);
547             }
548             finally
549             {
550                 synchronized(this)
551                 {
552                     _selecting=false;
553                 }
554             }
555         }
556 
557         /* ------------------------------------------------------------ */
558         public SelectorManager getManager()
559         {
560             return SelectorManager.this;
561         }
562 
563         /* ------------------------------------------------------------ */
564         public long getNow()
565         {
566             return _idleTimeout.getNow();
567         }
568         
569         /* ------------------------------------------------------------ */
570         public void scheduleIdle(Timeout.Task task)
571         {
572             if (_idleTimeout.getDuration() <= 0)
573                 return;
574             _idleTimeout.schedule(task);
575         }
576 
577         /* ------------------------------------------------------------ */
578         public void scheduleTimeout(Timeout.Task task, long timeoutMs)
579         {
580             _timeout.schedule(task, timeoutMs);
581         }
582         
583         /* ------------------------------------------------------------ */
584         public void cancelTimeout(Timeout.Task task)
585         {
586             task.cancel();
587         }
588 
589         /* ------------------------------------------------------------ */
590         public void wakeup()
591         {
592             Selector selector = _selector;
593             if (selector!=null)
594                 selector.wakeup();
595         }
596 
597         /* ------------------------------------------------------------ */
598         Selector getSelector()
599         {
600             return _selector;
601         }
602 
603         /* ------------------------------------------------------------ */
604         void stop() throws Exception
605         {
606             boolean selecting=true;
607             while(selecting)
608             {
609                 wakeup();
610                 synchronized (this)
611                 {
612                     selecting=_selecting;
613                 }
614             }
615             
616             ArrayList<SelectionKey> keys=new ArrayList<SelectionKey>(_selector.keys());
617             Iterator<SelectionKey> iter =keys.iterator();
618 
619             while (iter.hasNext())
620             {
621                 SelectionKey key = (SelectionKey)iter.next();
622                 if (key==null)
623                     continue;
624                 EndPoint endpoint = (EndPoint)key.attachment();
625                 if (endpoint!=null)
626                 {
627                     try
628                     {
629                         endpoint.close();
630                     }
631                     catch(IOException e)
632                     {
633                         Log.ignore(e);
634                     }
635                 }
636             }
637             
638             synchronized (this)
639             {
640                 _idleTimeout.cancelAll();
641                 _timeout.cancelAll();
642                 try
643                 {
644                     if (_selector != null)
645                         _selector.close();
646                 }
647                 catch (IOException e)
648                 {
649                     Log.ignore(e);
650                 } 
651                 _selector=null;
652             }
653         }
654     }
655 
656 }