1   package org.mortbay.io.nio;
2   
3   import java.io.IOException;
4   import java.nio.channels.CancelledKeyException;
5   import java.nio.channels.SelectionKey;
6   import java.nio.channels.Selector;
7   import java.nio.channels.ServerSocketChannel;
8   import java.nio.channels.SocketChannel;
9   import java.util.ArrayList;
10  import java.util.Iterator;
11  import java.util.List;
12  
13  import org.mortbay.component.AbstractLifeCycle;
14  import org.mortbay.component.LifeCycle;
15  import org.mortbay.io.Connection;
16  import org.mortbay.io.EndPoint;
17  import org.mortbay.log.Log;
18  import org.mortbay.thread.Timeout;
19  
20  
21  /* ------------------------------------------------------------ */
22  /**
23   * The Selector Manager manages and number of SelectSets to allow
24   * NIO scheduling to scale to large numbers of connections.
25   * 
26   * @author gregw
27   *
28   */
29  public abstract class SelectorManager extends AbstractLifeCycle
30  {
31      private long _maxIdleTime;
32      private long _lowResourcesConnections;
33      private long _lowResourcesMaxIdleTime;
34      private transient SelectSet[] _selectSet;
35      private int _selectSets=1;
36      private volatile int _set;
37      
38  
39      /* ------------------------------------------------------------ */
40      /**
41       * @param maxIdleTime The maximum period in milli seconds that a connection may be idle before it is closed.
42       * @see {@link #setLowResourcesMaxIdleTime(long)}
43       */
44      public void setMaxIdleTime(long maxIdleTime)
45      {
46          _maxIdleTime=maxIdleTime;
47      }
48      
49      /* ------------------------------------------------------------ */
50      /**
51       * @param selectSets
52       */
53      public void setSelectSets(int selectSets)
54      {
55          long lrc = _lowResourcesConnections * _selectSets; 
56          _selectSets=selectSets;
57          _lowResourcesConnections=lrc/_selectSets;
58      }
59      
60      /* ------------------------------------------------------------ */
61      /**
62       * @return
63       */
64      public long getMaxIdleTime()
65      {
66          return _maxIdleTime;
67      }
68      
69      /* ------------------------------------------------------------ */
70      /**
71       * @return
72       */
73      public int getSelectSets()
74      {
75          return _selectSets;
76      }
77      
78      /* ------------------------------------------------------------ */
79      /** Register a channel
80       * @param channel
81       * @param att Attached Object
82       * @throws IOException
83       */
84      public void register(SocketChannel channel, Object att) throws IOException
85      {
86          int s=_set++; 
87          s=s%_selectSets;
88          SelectSet[] sets=_selectSet;
89          if (sets!=null)
90          {
91              SelectSet set=sets[s];
92              set.addChange(channel,att);
93              set.wakeup();
94          }
95      }
96      
97      /* ------------------------------------------------------------ */
98      /** Register a serverchannel
99       * @param acceptChannel
100      * @return
101      * @throws IOException
102      */
103     public void register(ServerSocketChannel acceptChannel) throws IOException
104     {
105         int s=_set++; 
106         s=s%_selectSets;
107         SelectSet set=_selectSet[s];
108         set.addChange(acceptChannel);
109         set.wakeup();
110     }
111 
112     /* ------------------------------------------------------------ */
113     /**
114      * @return the lowResourcesConnections
115      */
116     public long getLowResourcesConnections()
117     {
118         return _lowResourcesConnections*_selectSets;
119     }
120 
121     /* ------------------------------------------------------------ */
122     /**
123      * Set the number of connections, which if exceeded places this manager in low resources state.
124      * This is not an exact measure as the connection count is averaged over the select sets.
125      * @param lowResourcesConnections the number of connections
126      * @see {@link #setLowResourcesMaxIdleTime(long)}
127      */
128     public void setLowResourcesConnections(long lowResourcesConnections)
129     {
130         _lowResourcesConnections=(lowResourcesConnections+_selectSets-1)/_selectSets;
131     }
132 
133     /* ------------------------------------------------------------ */
134     /**
135      * @return the lowResourcesMaxIdleTime
136      */
137     public long getLowResourcesMaxIdleTime()
138     {
139         return _lowResourcesMaxIdleTime;
140     }
141 
142     /* ------------------------------------------------------------ */
143     /**
144      * @param lowResourcesMaxIdleTime the period in ms that a connection is allowed to be idle when this SelectSet has more connections than {@link #getLowResourcesConnections()}
145      * @see {@link #setMaxIdleTime(long)}
146      */
147     public void setLowResourcesMaxIdleTime(long lowResourcesMaxIdleTime)
148     {
149         _lowResourcesMaxIdleTime=lowResourcesMaxIdleTime;
150     }
151     
152     /* ------------------------------------------------------------ */
153     /**
154      * @param acceptorID
155      * @throws IOException
156      */
157     public void doSelect(int acceptorID) throws IOException
158     {
159         SelectSet[] sets= _selectSet;
160         if (sets!=null && sets.length>acceptorID && sets[acceptorID]!=null)
161             sets[acceptorID].doSelect();
162     }
163 
164     /* ------------------------------------------------------------ */
165     /**
166      * @param key
167      * @return
168      * @throws IOException 
169      */
170     protected abstract SocketChannel acceptChannel(SelectionKey key) throws IOException;
171 
172     /* ------------------------------------------------------------------------------- */
173     public abstract boolean dispatch(Runnable task);
174 
175     /* ------------------------------------------------------------ */
176     /* (non-Javadoc)
177      * @see org.mortbay.component.AbstractLifeCycle#doStart()
178      */
179     protected void doStart() throws Exception
180     {
181         _selectSet = new SelectSet[_selectSets];
182         for (int i=0;i<_selectSet.length;i++)
183             _selectSet[i]= new SelectSet(i);
184 
185         super.doStart();
186     }
187 
188 
189     /* ------------------------------------------------------------------------------- */
190     protected void doStop() throws Exception
191     {
192         SelectSet[] sets= _selectSet;
193         _selectSet=null;
194         if (sets!=null)
195             for (int i=0;i<sets.length;i++)
196             {
197                 SelectSet set = sets[i];
198                 if (set!=null)
199                     set.stop();
200             }
201         super.doStop();
202     }
203 
204     /* ------------------------------------------------------------ */
205     /**
206      * @param endpoint
207      */
208     protected abstract void endPointClosed(SelectChannelEndPoint endpoint);
209 
210     /* ------------------------------------------------------------ */
211     /**
212      * @param endpoint
213      */
214     protected abstract void endPointOpened(SelectChannelEndPoint endpoint);
215 
216     /* ------------------------------------------------------------------------------- */
217     protected abstract Connection newConnection(SocketChannel channel, SelectChannelEndPoint endpoint);
218 
219     /* ------------------------------------------------------------ */
220     /**
221      * @param channel
222      * @param selectSet
223      * @param sKey
224      * @return
225      * @throws IOException
226      */
227     protected abstract SelectChannelEndPoint newEndPoint(SocketChannel channel, SelectorManager.SelectSet selectSet, SelectionKey sKey) throws IOException;
228 
229     /* ------------------------------------------------------------------------------- */
230     protected void connectionFailed(SocketChannel channel,Throwable ex,Object attachment)
231     {
232         Log.warn(ex);
233     }
234     
235     /* ------------------------------------------------------------------------------- */
236     /* ------------------------------------------------------------------------------- */
237     /* ------------------------------------------------------------------------------- */
238     public class SelectSet 
239     {
240         private transient int _change;
241         private transient List<Object>[] _changes;
242         private transient Timeout _idleTimeout;
243         private transient int _nextSet;
244         private transient Timeout _timeout;
245         private transient Selector _selector;
246         private transient int _setID;
247         private transient boolean _selecting;
248         private transient int _jvmBug;
249         
250         /* ------------------------------------------------------------ */
251         SelectSet(int acceptorID) throws Exception
252         {
253             _setID=acceptorID;
254 
255             _idleTimeout = new Timeout(this);
256             _idleTimeout.setDuration(getMaxIdleTime());
257             _timeout = new Timeout(this);
258             _timeout.setDuration(0L);
259 
260             // create a selector;
261             _selector = Selector.open();
262             _changes = new List[] {new ArrayList(),new ArrayList()};
263             _change=0;
264         }
265         
266         /* ------------------------------------------------------------ */
267         public void addChange(Object point)
268         {
269             synchronized (_changes)
270             {
271                 _changes[_change].add(point);
272                 if (point instanceof SocketChannel)
273                     _changes[_change].add(null);
274             }
275         }
276         
277         /* ------------------------------------------------------------ */
278         public void addChange(SocketChannel channel, Object att)
279         {   
280             synchronized (_changes)
281             {
282                 _changes[_change].add(channel);
283                 _changes[_change].add(att);
284             }
285         }
286         
287         /* ------------------------------------------------------------ */
288         public void cancelIdle(Timeout.Task task)
289         {
290             task.cancel();
291         }
292 
293         /* ------------------------------------------------------------ */
294         /**
295          * Select and dispatch tasks found from changes and the selector.
296          * 
297          * @throws IOException
298          */
299         public void doSelect() throws IOException
300         {
301             try
302             {
303                 List<?> changes;
304                 synchronized (_changes)
305                 {
306                     changes=_changes[_change];
307                     _change=_change==0?1:0;
308                     _selecting=true;
309                 }
310 
311                 // Make any key changes required
312                 for (int i = 0; i < changes.size(); i++)
313                 {
314                     try
315                     {
316                         Object o = changes.get(i);
317                         if (o instanceof EndPoint)
318                         {
319                             // Update the operations for a key.
320                             SelectChannelEndPoint endpoint = (SelectChannelEndPoint)o;
321                             endpoint.doUpdateKey();
322                         }
323                         else if (o instanceof Runnable)
324                         {
325                             dispatch((Runnable)o);
326                         }
327                         else if (o instanceof SocketChannel)
328                         {
329                             // finish accepting/connecting this connection
330                             SocketChannel channel=(SocketChannel)o;
331                             Object att = changes.get(++i);
332 
333                             if (channel.isConnected())
334                             {
335                                 SelectionKey key = channel.register(_selector,SelectionKey.OP_READ,att);
336                                 SelectChannelEndPoint endpoint = newEndPoint(channel,this,key);
337                                 key.attach(endpoint);
338                                 endpoint.schedule();
339                             }
340                             else
341                             {
342                                 channel.register(_selector,SelectionKey.OP_CONNECT,att);
343                             }
344 
345                         }
346                         else if (o instanceof ServerSocketChannel)
347                         {
348                             ServerSocketChannel channel = (ServerSocketChannel)o;
349                             channel.register(getSelector(),SelectionKey.OP_ACCEPT);
350                         }
351                         else
352                             throw new IllegalArgumentException(o.toString());
353                     }
354                     catch (CancelledKeyException e)
355                     {
356                         if (isRunning())
357                             Log.warn(e);
358                         else
359                             Log.debug(e);
360                     }
361                 }
362                 changes.clear();
363 
364                 long idle_next = 0;
365                 long retry_next = 0;
366                 long now=System.currentTimeMillis();
367                 synchronized (this)
368                 {
369                     _idleTimeout.setNow(now);
370                     _timeout.setNow(now);
371                     if (_lowResourcesConnections>0 && _selector.keys().size()>_lowResourcesConnections)
372                         _idleTimeout.setDuration(_lowResourcesMaxIdleTime);
373                     else 
374                         _idleTimeout.setDuration(_maxIdleTime);
375                     idle_next=_idleTimeout.getTimeToNext();
376                     retry_next=_timeout.getTimeToNext();
377                 }
378 
379                 // workout how low to wait in select
380                 long wait = 1000L;  // not getMaxIdleTime() as the now value of the idle timers needs to be updated.
381                 if (idle_next >= 0 && wait > idle_next)
382                     wait = idle_next;
383                 if (wait > 0 && retry_next >= 0 && wait > retry_next)
384                     wait = retry_next;
385     
386                 // Do the select.
387                 if (wait > 0) 
388                 {
389                     long before=now;
390                     int selected=_selector.select(wait);
391                     now = System.currentTimeMillis();
392                     _idleTimeout.setNow(now);
393                     _timeout.setNow(now);
394 
395                     // Look for JVM bug  http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6403933
396                     if (selected==0  && (now-before)<wait/2)
397                     {
398                         if (_jvmBug++>5) 
399                         {
400                             // Probably JVM BUG!    
401                             for (SelectionKey key: _selector.keys())
402                             {    
403                                 if (key.interestOps()==0 && key.isValid())
404                                     key.cancel();
405                             }
406                             _selector.selectNow();
407                         } 
408                     }
409                     else
410                         _jvmBug=0;
411                 }
412                 else 
413                 {
414                     _selector.selectNow();
415                     _jvmBug=0;
416                 }
417 
418                 // have we been destroyed while sleeping\
419                 if (_selector==null || !_selector.isOpen())
420                     return;
421 
422                 // Look for things to do
423                 for (SelectionKey key: _selector.selectedKeys())
424                 {   
425                     try
426                     {
427                         if (!key.isValid())
428                         {
429                             key.cancel();
430                             SelectChannelEndPoint endpoint = (SelectChannelEndPoint)key.attachment();
431                             if (endpoint != null)
432                                 endpoint.doUpdateKey();
433                             continue;
434                         }
435 
436                         Object att = key.attachment();
437                         if (att instanceof SelectChannelEndPoint)
438                         {
439                             ((SelectChannelEndPoint)att).schedule();
440                         }
441                         else if (key.isAcceptable())
442                         {
443                             SocketChannel channel = acceptChannel(key);
444                             if (channel==null)
445                                 continue;
446 
447                             channel.configureBlocking(false);
448 
449                             // TODO make it reluctant to leave 0
450                             _nextSet=++_nextSet%_selectSet.length;
451 
452                             // Is this for this selectset
453                             if (_nextSet==_setID)
454                             {
455                                 // bind connections to this select set.
456                                 SelectionKey cKey = channel.register(_selectSet[_nextSet].getSelector(), SelectionKey.OP_READ);
457                                 SelectChannelEndPoint endpoint=newEndPoint(channel,_selectSet[_nextSet],cKey);
458                                 cKey.attach(endpoint);
459                                 if (endpoint != null)
460                                     endpoint.schedule();
461                             }
462                             else
463                             {
464                                 // nope - give it to another.
465                                 _selectSet[_nextSet].addChange(channel);
466                                 _selectSet[_nextSet].wakeup();
467                             }
468                         }
469                         else if (key.isConnectable())
470                         {
471                             // Complete a connection of a registered channel
472                             SocketChannel channel = (SocketChannel)key.channel();
473                             boolean connected=false;
474                             try
475                             {
476                                 connected=channel.finishConnect();
477                             }
478                             catch(Exception e)
479                             {
480                                 connectionFailed(channel,e,att);
481                             }
482                             finally
483                             {
484                                 if (connected)
485                                 {
486                                     key.interestOps(SelectionKey.OP_READ);
487                                     SelectChannelEndPoint endpoint = newEndPoint(channel,this,key);
488                                     key.attach(endpoint);
489                                     endpoint.schedule();
490                                 }
491                                 else
492                                 {
493                                     key.cancel();
494                                 }
495                             }
496                         }
497                         else
498                         {
499                             // Wrap readable registered channel in an endpoint
500                             SocketChannel channel = (SocketChannel)key.channel();
501                             SelectChannelEndPoint endpoint = newEndPoint(channel,this,key);
502                             key.attach(endpoint);
503                             if (key.isReadable())
504                                 endpoint.schedule();                           
505                         }
506                         key = null;
507                     }
508                     catch (CancelledKeyException e)
509                     {
510                         Log.ignore(e);
511                     }
512                     catch (Exception e)
513                     {
514                         if (isRunning())
515                             Log.warn(e);
516                         else
517                             Log.ignore(e);
518 
519                         if (key != null && !(key.channel() instanceof ServerSocketChannel) && key.isValid())
520                             key.cancel();
521                     }
522                 }
523                 
524                 // Everything always handled
525                 _selector.selectedKeys().clear();
526 
527                 // tick over the timers
528                 _idleTimeout.tick();
529                 _timeout.tick();
530 
531             }
532             catch (CancelledKeyException e)
533             {
534                 Log.ignore(e);
535             }
536             finally
537             {
538                 synchronized(this)
539                 {
540                     _selecting=false;
541                 }
542             }
543         }
544 
545         /* ------------------------------------------------------------ */
546         public SelectorManager getManager()
547         {
548             return SelectorManager.this;
549         }
550 
551         /* ------------------------------------------------------------ */
552         public long getNow()
553         {
554             return _idleTimeout.getNow();
555         }
556         
557         /* ------------------------------------------------------------ */
558         public void scheduleIdle(Timeout.Task task)
559         {
560             if (_idleTimeout.getDuration() <= 0)
561                 return;
562             _idleTimeout.schedule(task);
563         }
564 
565         /* ------------------------------------------------------------ */
566         public void scheduleTimeout(Timeout.Task task, long timeoutMs)
567         {
568             _timeout.schedule(task, timeoutMs);
569         }
570         
571         /* ------------------------------------------------------------ */
572         public void cancelTimeout(Timeout.Task task)
573         {
574             task.cancel();
575         }
576 
577         /* ------------------------------------------------------------ */
578         public void wakeup()
579         {
580             Selector selector = _selector;
581             if (selector!=null)
582                 selector.wakeup();
583         }
584 
585         /* ------------------------------------------------------------ */
586         Selector getSelector()
587         {
588             return _selector;
589         }
590 
591         /* ------------------------------------------------------------ */
592         void stop() throws Exception
593         {
594             boolean selecting=true;
595             while(selecting)
596             {
597                 wakeup();
598                 synchronized (this)
599                 {
600                     selecting=_selecting;
601                 }
602             }
603             
604             ArrayList<SelectionKey> keys=new ArrayList<SelectionKey>(_selector.keys());
605             Iterator<SelectionKey> iter =keys.iterator();
606 
607             while (iter.hasNext())
608             {
609                 SelectionKey key = (SelectionKey)iter.next();
610                 if (key==null)
611                     continue;
612                 EndPoint endpoint = (EndPoint)key.attachment();
613                 if (endpoint!=null)
614                 {
615                     try
616                     {
617                         endpoint.close();
618                     }
619                     catch(IOException e)
620                     {
621                         Log.ignore(e);
622                     }
623                 }
624             }
625             
626             synchronized (this)
627             {
628                 _idleTimeout.cancelAll();
629                 _timeout.cancelAll();
630                 try
631                 {
632                     if (_selector != null)
633                         _selector.close();
634                 }
635                 catch (IOException e)
636                 {
637                     Log.ignore(e);
638                 } 
639                 _selector=null;
640             }
641         }
642     }
643 
644 }