View Javadoc

1   //========================================================================
2   //Copyright 2004-2008 Mort Bay Consulting Pty. Ltd.
3   //------------------------------------------------------------------------
4   //Licensed under the Apache License, Version 2.0 (the "License");
5   //you may not use this file except in compliance with the License.
6   //You may obtain a copy of the License at 
7   //http://www.apache.org/licenses/LICENSE-2.0
8   //Unless required by applicable law or agreed to in writing, software
9   //distributed under the License is distributed on an "AS IS" BASIS,
10  //WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11  //See the License for the specific language governing permissions and
12  //limitations under the License.
13  //========================================================================
14  
15  package org.mortbay.io.nio;
16  
17  import java.io.IOException;
18  import java.nio.channels.ByteChannel;
19  import java.nio.channels.CancelledKeyException;
20  import java.nio.channels.SelectableChannel;
21  import java.nio.channels.SelectionKey;
22  import java.nio.channels.Selector;
23  import java.nio.channels.ServerSocketChannel;
24  import java.nio.channels.SocketChannel;
25  import java.util.ArrayList;
26  import java.util.Iterator;
27  import java.util.List;
28  
29  import org.mortbay.component.AbstractLifeCycle;
30  import org.mortbay.component.LifeCycle;
31  import org.mortbay.io.Connection;
32  import org.mortbay.io.EndPoint;
33  import org.mortbay.log.Log;
34  import org.mortbay.thread.Timeout;
35  
36  
37  /* ------------------------------------------------------------ */
38  /**
39   * The Selector Manager manages and number of SelectSets to allow
40   * NIO scheduling to scale to large numbers of connections.
41   * 
42   * @author gregw
43   *
44   */
45  public abstract class SelectorManager extends AbstractLifeCycle
46  {
47      private static final int __JVMBUG_THRESHHOLD=Integer.getInteger("org.mortbay.io.nio.JVMBUG_THRESHHOLD",32).intValue();
48      private boolean _delaySelectKeyUpdate=true;
49      private long _maxIdleTime;
50      private long _lowResourcesConnections;
51      private long _lowResourcesMaxIdleTime;
52      private transient SelectSet[] _selectSet;
53      private int _selectSets=1;
54      private volatile int _set;
55      private boolean _jvmBug0;
56      private boolean _jvmBug1;
57      
58  
59      /* ------------------------------------------------------------ */
60      /**
61       * @param maxIdleTime The maximum period in milli seconds that a connection may be idle before it is closed.
62       * @see {@link #setLowResourcesMaxIdleTime(long)}
63       */
64      public void setMaxIdleTime(long maxIdleTime)
65      {
66          _maxIdleTime=maxIdleTime;
67      }
68      
69      /* ------------------------------------------------------------ */
70      /**
71       * @param selectSets
72       */
73      public void setSelectSets(int selectSets)
74      {
75          long lrc = _lowResourcesConnections * _selectSets; 
76          _selectSets=selectSets;
77          _lowResourcesConnections=lrc/_selectSets;
78      }
79      
80      /* ------------------------------------------------------------ */
81      /**
82       * @return
83       */
84      public long getMaxIdleTime()
85      {
86          return _maxIdleTime;
87      }
88      
89      /* ------------------------------------------------------------ */
90      /**
91       * @return
92       */
93      public int getSelectSets()
94      {
95          return _selectSets;
96      }
97      
98      /* ------------------------------------------------------------ */
99      /**
100      * @return
101      */
102     public boolean isDelaySelectKeyUpdate()
103     {
104         return _delaySelectKeyUpdate;
105     }
106 
107     /* ------------------------------------------------------------ */
108     /** Register a channel
109      * @param channel
110      * @param att Attached Object
111      * @throws IOException
112      */
113     public void register(SocketChannel channel, Object att) throws IOException
114     {
115         int s=_set++; 
116         s=s%_selectSets;
117         SelectSet[] sets=_selectSet;
118         if (sets!=null)
119         {
120             SelectSet set=sets[s];
121             set.addChange(channel,att);
122             set.wakeup();
123         }
124     }
125     
126     /* ------------------------------------------------------------ */
127     /** Register a serverchannel
128      * @param acceptChannel
129      * @return
130      * @throws IOException
131      */
132     public void register(ServerSocketChannel acceptChannel) throws IOException
133     {
134         int s=_set++; 
135         s=s%_selectSets;
136         SelectSet set=_selectSet[s];
137         set.addChange(acceptChannel);
138         set.wakeup();
139     }
140 
141     /* ------------------------------------------------------------ */
142     /**
143      * @return the lowResourcesConnections
144      */
145     public long getLowResourcesConnections()
146     {
147         return _lowResourcesConnections*_selectSets;
148     }
149 
150     /* ------------------------------------------------------------ */
151     /**
152      * Set the number of connections, which if exceeded places this manager in low resources state.
153      * This is not an exact measure as the connection count is averaged over the select sets.
154      * @param lowResourcesConnections the number of connections
155      * @see {@link #setLowResourcesMaxIdleTime(long)}
156      */
157     public void setLowResourcesConnections(long lowResourcesConnections)
158     {
159         _lowResourcesConnections=(lowResourcesConnections+_selectSets-1)/_selectSets;
160     }
161 
162     /* ------------------------------------------------------------ */
163     /**
164      * @return the lowResourcesMaxIdleTime
165      */
166     public long getLowResourcesMaxIdleTime()
167     {
168         return _lowResourcesMaxIdleTime;
169     }
170 
171     /* ------------------------------------------------------------ */
172     /**
173      * @param lowResourcesMaxIdleTime the period in ms that a connection is allowed to be idle when this SelectSet has more connections than {@link #getLowResourcesConnections()}
174      * @see {@link #setMaxIdleTime(long)}
175      */
176     public void setLowResourcesMaxIdleTime(long lowResourcesMaxIdleTime)
177     {
178         _lowResourcesMaxIdleTime=lowResourcesMaxIdleTime;
179     }
180     
181     /* ------------------------------------------------------------ */
182     /**
183      * @param acceptorID
184      * @throws IOException
185      */
186     public void doSelect(int acceptorID) throws IOException
187     {
188         SelectSet[] sets= _selectSet;
189         if (sets!=null && sets.length>acceptorID && sets[acceptorID]!=null)
190             sets[acceptorID].doSelect();
191     }
192 
193 
194     /* ------------------------------------------------------------ */
195     /**
196      * @param delaySelectKeyUpdate
197      */
198     public void setDelaySelectKeyUpdate(boolean delaySelectKeyUpdate)
199     {
200         _delaySelectKeyUpdate=delaySelectKeyUpdate;
201     }
202 
203     /* ------------------------------------------------------------ */
204     /**
205      * @param key
206      * @return
207      * @throws IOException 
208      */
209     protected abstract SocketChannel acceptChannel(SelectionKey key) throws IOException;
210 
211     /* ------------------------------------------------------------------------------- */
212     public abstract boolean dispatch(Runnable task) throws IOException;
213 
214     /* ------------------------------------------------------------ */
215     /* (non-Javadoc)
216      * @see org.mortbay.component.AbstractLifeCycle#doStart()
217      */
218     protected void doStart() throws Exception
219     {
220         _selectSet = new SelectSet[_selectSets];
221         for (int i=0;i<_selectSet.length;i++)
222             _selectSet[i]= new SelectSet(i);
223 
224         super.doStart();
225     }
226 
227 
228     /* ------------------------------------------------------------------------------- */
229     protected void doStop() throws Exception
230     {
231         SelectSet[] sets= _selectSet;
232         _selectSet=null;
233         if (sets!=null)
234             for (int i=0;i<sets.length;i++)
235             {
236                 SelectSet set = sets[i];
237                 if (set!=null)
238                     set.stop();
239             }
240         super.doStop();
241     }
242 
243     /* ------------------------------------------------------------ */
244     /**
245      * @param endpoint
246      */
247     protected abstract void endPointClosed(SelectChannelEndPoint endpoint);
248 
249     /* ------------------------------------------------------------ */
250     /**
251      * @param endpoint
252      */
253     protected abstract void endPointOpened(SelectChannelEndPoint endpoint);
254 
255     /* ------------------------------------------------------------------------------- */
256     protected abstract Connection newConnection(SocketChannel channel, SelectChannelEndPoint endpoint);
257 
258     /* ------------------------------------------------------------ */
259     /**
260      * @param channel
261      * @param selectSet
262      * @param sKey
263      * @return
264      * @throws IOException
265      */
266     protected abstract SelectChannelEndPoint newEndPoint(SocketChannel channel, SelectorManager.SelectSet selectSet, SelectionKey sKey) throws IOException;
267 
268     /* ------------------------------------------------------------------------------- */
269     protected void connectionFailed(SocketChannel channel,Throwable ex,Object attachment)
270     {
271         Log.warn(ex);
272     }
273     
274     /* ------------------------------------------------------------------------------- */
275     /* ------------------------------------------------------------------------------- */
276     /* ------------------------------------------------------------------------------- */
277     public class SelectSet 
278     {
279         private transient int _change;
280         private transient List[] _changes;
281         private transient Timeout _idleTimeout;
282         private transient int _nextSet;
283         private transient Timeout _retryTimeout;
284         private transient Selector _selector;
285         private transient int _setID;
286         private transient int _jvmBug;
287         private volatile boolean _selecting;
288         
289         /* ------------------------------------------------------------ */
290         SelectSet(int acceptorID) throws Exception
291         {
292             _setID=acceptorID;
293 
294             _idleTimeout = new Timeout(this);
295             _idleTimeout.setDuration(getMaxIdleTime());
296             _retryTimeout = new Timeout(this);
297             _retryTimeout.setDuration(0L);
298 
299             // create a selector;
300             _selector = Selector.open();
301             _changes = new ArrayList[] {new ArrayList(),new ArrayList()};
302             _change=0;
303         }
304         
305         /* ------------------------------------------------------------ */
306         public void addChange(Object point)
307         {
308             synchronized (_changes)
309             {
310                 _changes[_change].add(point);
311             }
312         }
313         
314         /* ------------------------------------------------------------ */
315         public void addChange(SelectableChannel channel, Object att)
316         {   
317             if (att==null)
318                 addChange(channel);
319             else if (att instanceof EndPoint)
320                 addChange(att);
321             else
322                 addChange(new ChangeSelectableChannel(channel,att));
323         }
324         
325         /* ------------------------------------------------------------ */
326         public void cancelIdle(Timeout.Task task)
327         {
328             synchronized (this)
329             {
330                 task.cancel();
331             }
332         }
333 
334         /* ------------------------------------------------------------ */
335         /**
336          * Select and dispatch tasks found from changes and the selector.
337          * 
338          * @throws IOException
339          */
340         public void doSelect() throws IOException
341         {
342             SelectionKey key=null;
343             
344             try
345             {
346                 List changes;
347                 final Selector selector;
348                 synchronized (_changes)
349                 {
350                     changes=_changes[_change];
351                     _change=_change==0?1:0;
352                     _selecting=true;
353                     selector=_selector;
354                 }
355                 
356 
357                 // Make any key changes required
358                 for (int i = 0; i < changes.size(); i++)
359                 {
360                     try
361                     {
362                         Object o = changes.get(i);
363                         
364                         if (o instanceof EndPoint)
365                         {
366                             // Update the operations for a key.
367                             SelectChannelEndPoint endpoint = (SelectChannelEndPoint)o;
368                             endpoint.doUpdateKey();
369                         }
370                         else if (o instanceof Runnable)
371                         {
372                             dispatch((Runnable)o);
373                         }
374                         else if (o instanceof ChangeSelectableChannel)
375                         {
376                             // finish accepting/connecting this connection
377                             final ChangeSelectableChannel asc = (ChangeSelectableChannel)o;
378                             final SelectableChannel channel=asc._channel;
379                             final Object att = asc._attachment;
380 
381                             if ((channel instanceof SocketChannel) && ((SocketChannel)channel).isConnected())
382                             {
383                                 key = channel.register(selector,SelectionKey.OP_READ,att);
384                                 SelectChannelEndPoint endpoint = newEndPoint((SocketChannel)channel,this,key);
385                                 key.attach(endpoint);
386                                 endpoint.dispatch();
387                             }
388                             else if (channel.isOpen())
389                             {
390                                 channel.register(selector,SelectionKey.OP_CONNECT,att);
391                             }
392                         }
393                         else if (o instanceof SocketChannel)
394                         {
395                             final SocketChannel channel=(SocketChannel)o;
396 
397                             if (channel.isConnected())
398                             {
399                                 key = channel.register(selector,SelectionKey.OP_READ,null);
400                                 SelectChannelEndPoint endpoint = newEndPoint(channel,this,key);
401                                 key.attach(endpoint);
402                                 endpoint.dispatch();
403                             }
404                             else
405                             {
406                                 channel.register(selector,SelectionKey.OP_CONNECT,null);
407                             }
408                         }
409                         else if (o instanceof ServerSocketChannel)
410                         {
411                             ServerSocketChannel channel = (ServerSocketChannel)o;
412                             channel.register(getSelector(),SelectionKey.OP_ACCEPT);
413                         }
414                         else if (o instanceof ChangeTask)
415                         {
416                             ((ChangeTask)o).run();
417                         }
418                         else
419                             throw new IllegalArgumentException(o.toString());
420                     }
421                     catch (CancelledKeyException e)
422                     {
423                         if (isRunning())
424                             Log.warn(e);
425                         else
426                             Log.debug(e);
427                     }
428                 }
429                 changes.clear();
430 
431                 long idle_next = 0;
432                 long retry_next = 0;
433                 long now=System.currentTimeMillis();
434                 synchronized (this)
435                 {
436                     _idleTimeout.setNow(now);
437                     _retryTimeout.setNow(now);
438                     if (_lowResourcesConnections>0 && selector.keys().size()>_lowResourcesConnections)
439                         _idleTimeout.setDuration(_lowResourcesMaxIdleTime);
440                     else 
441                         _idleTimeout.setDuration(_maxIdleTime);
442                     idle_next=_idleTimeout.getTimeToNext();
443                     retry_next=_retryTimeout.getTimeToNext();
444                 }
445 
446                 // workout how low to wait in select
447                 long wait = 1000L;  // not getMaxIdleTime() as the now value of the idle timers needs to be updated.
448                 if (idle_next >= 0 && wait > idle_next)
449                     wait = idle_next;
450                 if (wait > 0 && retry_next >= 0 && wait > retry_next)
451                     wait = retry_next;
452     
453                 // Do the select.
454                 if (wait > 10) // TODO tune or configure this
455                 {
456                     long before=now;
457                     int selected=selector.select(wait);
458                     now = System.currentTimeMillis();
459                     _idleTimeout.setNow(now);
460                     _retryTimeout.setNow(now);
461 
462                     // Look for JVM bugs
463                     // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6403933
464                     // http://bugs.sun.com/view_bug.do?bug_id=6693490
465                     if (selected==0 && wait>0 && (now-before)<wait/2 && __JVMBUG_THRESHHOLD>0)
466                     {
467                         _jvmBug++;
468                         if (_jvmBug>(__JVMBUG_THRESHHOLD*2))
469                         {
470                             synchronized (this)
471                             {
472                                 // BLOODY SUN BUG !!!  Try refreshing the entire selector.
473                                 if (_jvmBug1)
474                                     Log.debug("seeing JVM BUG(s) - recreating selector");
475                                 else
476                                 {
477                                     _jvmBug1=true;
478                                     Log.info("seeing JVM BUG(s) - recreating selector");
479                                 }
480                                 
481                                 final Selector new_selector = Selector.open();
482                                 Iterator iterator = _selector.keys().iterator();
483                                 while (iterator.hasNext())
484                                 {
485                                     SelectionKey k = (SelectionKey)iterator.next();
486                                     final SelectableChannel channel = k.channel();
487                                     final Object attachment = k.attachment();
488                                     
489                                     k.cancel();
490                                     if (attachment==null)
491                                         addChange(channel);
492                                     else
493                                         addChange(channel,attachment);
494                                 }
495                                 _selector.close();
496                                 _selector=new_selector;
497                                 return;
498                             }
499                         }
500                         else if (_jvmBug>__JVMBUG_THRESHHOLD)
501                         {
502                             // Cancel keys with 0 interested ops
503                             if (_jvmBug0)
504                                 Log.debug("seeing JVM BUG(s) - cancelling interestOps==0");
505                             else
506                             {
507                                 _jvmBug0=true;
508                                 Log.info("seeing JVM BUG(s) - cancelling interestOps==0");
509                             }
510                             
511                             Iterator iter = selector.keys().iterator();
512                             while(iter.hasNext())
513                             {
514                                 SelectionKey k = (SelectionKey) iter.next();
515                                 if (k.isValid()&&k.interestOps()==0)
516                                 {
517                                     k.cancel();
518                                 }
519                             }
520                             return;
521                         }
522                     }
523                     else
524                         _jvmBug=0;
525                 }
526                 else 
527                 {
528                     selector.selectNow();
529                     _jvmBug=0;
530                 }
531 
532                 // have we been destroyed while sleeping
533                 if (_selector==null || !selector.isOpen())
534                     return;
535 
536                 // Look for things to do
537                 Iterator iter = selector.selectedKeys().iterator();
538                 while (iter.hasNext())
539                 {
540                     key = (SelectionKey) iter.next();
541                                         
542                     try
543                     {
544                         if (!key.isValid())
545                         {
546                             key.cancel();
547                             SelectChannelEndPoint endpoint = (SelectChannelEndPoint)key.attachment();
548                             if (endpoint != null)
549                                 endpoint.doUpdateKey();
550                             continue;
551                         }
552 
553                         Object att = key.attachment();
554                         if (att instanceof SelectChannelEndPoint)
555                         {
556                             SelectChannelEndPoint endpoint = (SelectChannelEndPoint)att;
557                             endpoint.dispatch();
558                         }
559                         else if (key.isAcceptable())
560                         {
561                             SocketChannel channel = acceptChannel(key);
562                             if (channel==null)
563                                 continue;
564 
565                             channel.configureBlocking(false);
566 
567                             // TODO make it reluctant to leave 0
568                             _nextSet=++_nextSet%_selectSet.length;
569 
570                             // Is this for this selectset
571                             if (_nextSet==_setID)
572                             {
573                                 // bind connections to this select set.
574                                 SelectionKey cKey = channel.register(_selectSet[_nextSet].getSelector(), SelectionKey.OP_READ);
575                                 SelectChannelEndPoint endpoint=newEndPoint(channel,_selectSet[_nextSet],cKey);
576                                 cKey.attach(endpoint);
577                                 if (endpoint != null)
578                                     endpoint.dispatch();
579                             }
580                             else
581                             {
582                                 // nope - give it to another.
583                                 _selectSet[_nextSet].addChange(channel);
584                                 _selectSet[_nextSet].wakeup();
585                             }
586                         }
587                         else if (key.isConnectable())
588                         {
589                             // Complete a connection of a registered channel
590                             SocketChannel channel = (SocketChannel)key.channel();
591                             boolean connected=false;
592                             try
593                             {
594                                 connected=channel.finishConnect();
595                             }
596                             catch(Exception e)
597                             {
598                                 connectionFailed(channel,e,att);
599                             }
600                             finally
601                             {
602                                 if (connected)
603                                 {
604                                     key.interestOps(SelectionKey.OP_READ);
605                                     SelectChannelEndPoint endpoint = newEndPoint(channel,this,key);
606                                     key.attach(endpoint);
607                                     endpoint.dispatch();
608                                 }
609                                 else
610                                 {
611                                     key.cancel();
612                                 }
613                             }
614                         }
615                         else
616                         {
617                             // Wrap readable registered channel in an endpoint
618                             SocketChannel channel = (SocketChannel)key.channel();
619                             SelectChannelEndPoint endpoint = newEndPoint(channel,this,key);
620                             key.attach(endpoint);
621                             if (key.isReadable())
622                                 endpoint.dispatch();                           
623                         }
624                         key = null;
625                     }
626                     catch (CancelledKeyException e)
627                     {
628                         Log.ignore(e);
629                     }
630                     catch (Exception e)
631                     {
632                         if (isRunning())
633                             Log.warn(e);
634                         else
635                             Log.ignore(e);
636 
637                         if (key != null && !(key.channel() instanceof ServerSocketChannel) && key.isValid())
638                         {
639                             key.interestOps(0);
640 
641                             key.cancel();
642                         } 
643                     }
644                 }
645                 
646                 // Everything always handled
647                 selector.selectedKeys().clear();
648 
649                 // tick over the timers
650                 _idleTimeout.tick(now);
651                 _retryTimeout.tick(now);
652                 
653             }
654             catch (CancelledKeyException e)
655             {
656                 Log.ignore(e);
657             }
658             finally
659             {
660                 _selecting=false;
661             }
662         }
663 
664         /* ------------------------------------------------------------ */
665         public SelectorManager getManager()
666         {
667             return SelectorManager.this;
668         }
669 
670         /* ------------------------------------------------------------ */
671         public long getNow()
672         {
673             return _idleTimeout.getNow();
674         }
675         
676         /* ------------------------------------------------------------ */
677         public void scheduleIdle(Timeout.Task task)
678         {
679             synchronized (this)
680             {
681                 if (_idleTimeout.getDuration() <= 0)
682                     return;
683                 
684                 task.schedule(_idleTimeout);
685             }
686         }
687 
688         /* ------------------------------------------------------------ */
689         public void scheduleTimeout(Timeout.Task task, long timeout)
690         {
691             synchronized (this)
692             {
693                 _retryTimeout.schedule(task, timeout);
694             }
695         }
696 
697         /* ------------------------------------------------------------ */
698         public void wakeup()
699         {
700             Selector selector = _selector;
701             if (selector!=null)
702                 selector.wakeup();
703         }
704 
705         /* ------------------------------------------------------------ */
706         Selector getSelector()
707         {
708             return _selector;
709         }
710 
711         /* ------------------------------------------------------------ */
712         void stop() throws Exception
713         {
714             boolean selecting=true;
715             while(selecting)
716             {
717                 wakeup();
718                 selecting=_selecting;
719             }
720             
721             ArrayList keys=new ArrayList(_selector.keys());
722             Iterator iter =keys.iterator();
723 
724             while (iter.hasNext())
725             {
726                 SelectionKey key = (SelectionKey)iter.next();
727                 if (key==null)
728                     continue;
729                 EndPoint endpoint = (EndPoint)key.attachment();
730                 if (endpoint!=null)
731                 {
732                     try
733                     {
734                         endpoint.close();
735                     }
736                     catch(IOException e)
737                     {
738                         Log.ignore(e);
739                     }
740                 }
741             }
742             
743             synchronized (this)
744             {
745                 selecting=_selecting;
746                 while(selecting)
747                 {
748                     wakeup();
749                     selecting=_selecting;
750                 }
751                 
752                 _idleTimeout.cancelAll();
753                 _retryTimeout.cancelAll();
754                 try
755                 {
756                     if (_selector != null)
757                         _selector.close();
758                 }
759                 catch (IOException e)
760                 {
761                     Log.ignore(e);
762                 } 
763                 _selector=null;
764             }
765         }
766     }
767 
768     /* ------------------------------------------------------------ */
769     private static class ChangeSelectableChannel
770     {
771         final SelectableChannel _channel;
772         final Object _attachment;
773         
774         public ChangeSelectableChannel(SelectableChannel channel, Object attachment)
775         {
776             super();
777             _channel = channel;
778             _attachment = attachment;
779         }
780     }
781 
782     /* ------------------------------------------------------------ */
783     private interface ChangeTask
784     {
785         public void run();
786     }
787 }