Console and Eepsite Jetty:

Switch back to QueuedThreadPool (ticket #1395)
In Jetty 5/6, the default QTP was not concurrent, so we switched to
ThreadPoolExecutor with a fixed-size queue, a set maxThreads,
and a RejectedExecutionPolicy of CallerRuns.
Unfortunately, CallerRuns causes lockups in Jetty NIO.
In addition, no flavor of TPE gives us what QTP does:
- TPE direct handoff (which we were using) never queues.
  This doesn't provide any burst management when maxThreads is reached.
  CallerRuns was an attempt to work around that.
- TPE unbounded queue does not adjust the number of threads.
  This doesn't provide automatic resource management.
- TPE bounded queue does not add threads until the queue is full.
  This doesn't provide good responsiveness to even small bursts.
QTP adds threads as soon as the queue is non-empty.
QTP as of Jetty 7 uses concurrent.
QTP unbounded queue is the default in Jetty.
So switch back to QTP with a bounded queue, which does what we want,
which is first expand the thread pool, then start queueing, then reject.

ref:
http://docs.oracle.com/javase/6/docs/api/java/util/concurrent/ThreadPoolExecutor.html
https://wiki.eclipse.org/Jetty/Howto/High_Load

apps/routerconsole/java/src/net/i2p/router/web/RouterConsoleRunner.java

@@ -15,11 +15,7 @@ import java.util.Map;
 import java.util.Properties;
 import java.util.SortedSet;
 import java.util.StringTokenizer;
-import java.util.concurrent.Executors;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.LinkedBlockingQueue;
 
 import net.i2p.I2PAppContext;
 import net.i2p.app.ClientAppManager;
@@ -122,7 +118,9 @@ public class RouterConsoleRunner implements RouterApp {
     private static final String USAGE = "Bad RouterConsoleRunner arguments, check clientApp.0.args in your clients.config file! " +
                                         "Usage: [[port host[,host]] [-s sslPort [host[,host]]] [webAppsDir]]";
 
+    /** this is for the handlers only. We will adjust for the connectors and acceptors below. */
     private static final int MIN_THREADS = 1;
+    /** this is for the handlers only. We will adjust for the connectors and acceptors below. */
     private static final int MAX_THREADS = 24;
     private static final int MAX_IDLE_TIME = 90*1000;
     private static final String THREAD_NAME = "RouterConsole Jetty";
@@ -318,19 +316,45 @@ public class RouterConsoleRunner implements RouterApp {
         _server = new Server();
         _server.setGracefulShutdown(1000);
 
-        try {
-            ThreadPool ctp = new CustomThreadPoolExecutor();
-            // Gone in Jetty 7
-            //ctp.prestartAllCoreThreads();
-            _server.setThreadPool(ctp);
-        } catch (Throwable t) {
+        // In Jetty 6, QTP was not concurrent, so we switched to
+        // ThreadPoolExecutor with a fixed-size queue, a set maxThreads,
+        // and a RejectedExecutionPolicy of CallerRuns.
+        // Unfortunately, CallerRuns causes lockups in Jetty NIO (ticket #1395)
+        // In addition, no flavor of TPE gives us what QTP does:
+        // - TPE direct handoff (which we were using) never queues.
+        //   This doesn't provide any burst management when maxThreads is reached.
+        //   CallerRuns was an attempt to work around that.
+        // - TPE unbounded queue does not adjust the number of threads.
+        //   This doesn't provide automatic resource management.
+        // - TPE bounded queue does not add threads until the queue is full.
+        //   This doesn't provide good responsiveness to even small bursts.
+        // QTP adds threads as soon as the queue is non-empty.
+        // QTP as of Jetty 7 uses concurrent.
+        // QTP unbounded queue is the default in Jetty.
+        // So switch back to QTP with a bounded queue.
+        //
+        // ref:
+        // http://docs.oracle.com/javase/6/docs/api/java/util/concurrent/ThreadPoolExecutor.html
+        // https://wiki.eclipse.org/Jetty/Howto/High_Load
+        //
+        //try {
+        //    ThreadPool ctp = new CustomThreadPoolExecutor();
+        //    // Gone in Jetty 7
+        //    //ctp.prestartAllCoreThreads();
+        //    _server.setThreadPool(ctp);
+        //} catch (Throwable t) {
             // class not found...
-            System.out.println("INFO: Jetty concurrent ThreadPool unavailable, using QueuedThreadPool");
-            QueuedThreadPool qtp = new QueuedThreadPool(MAX_THREADS);
-            qtp.setMinThreads(MIN_THREADS);
+            //System.out.println("INFO: Jetty concurrent ThreadPool unavailable, using QueuedThreadPool");
+            LinkedBlockingQueue<Runnable> lbq = new LinkedBlockingQueue<Runnable>(4*MAX_THREADS);
+            QueuedThreadPool qtp = new QueuedThreadPool(lbq);
+            // min and max threads will be set below
+            //qtp.setMinThreads(MIN_THREADS);
+            //qtp.setMaxThreads(MAX_THREADS);
             qtp.setMaxIdleTimeMs(MAX_IDLE_TIME);
+            qtp.setName(THREAD_NAME);
+            qtp.setDaemon(true);
             _server.setThreadPool(qtp);
-        }
+        //}
 
         HandlerCollection hColl = new HandlerCollection();
         ContextHandlerCollection chColl = new ContextHandlerCollection();
@@ -527,6 +551,10 @@ public class RouterConsoleRunner implements RouterApp {
                 System.err.println("Unable to bind routerconsole to any address on port " + _listenPort + (sslPort > 0 ? (" or SSL port " + sslPort) : ""));
                 return;
             }
+            // Each address spawns a Connector and an Acceptor thread
+            // If the min is less than this, we have no thread for the handlers or the expiration thread.
+            qtp.setMinThreads(MIN_THREADS + (2 * boundAddresses));
+            qtp.setMaxThreads(MAX_THREADS + (2 * boundAddresses));
 
             File tmpdir = new SecureDirectory(workDir, ROUTERCONSOLE + "-" +
                                                        (_listenPort != null ? _listenPort : _sslListenPort));
@@ -838,6 +866,7 @@ public class RouterConsoleRunner implements RouterApp {
      * Just to set the name and set Daemon
      * @since Jetty 6
      */
+/*****
     private static class CustomThreadPoolExecutor extends ExecutorThreadPool {
         public CustomThreadPoolExecutor() {
              super(new ThreadPoolExecutor(
@@ -848,11 +877,13 @@ public class RouterConsoleRunner implements RouterApp {
                   );
         }
     }
+*****/
 
     /**
      * Just to set the name and set Daemon
      * @since Jetty 6
      */
+/*****
     private static class CustomThreadFactory implements ThreadFactory {
 
         public Thread newThread(Runnable r) {
@@ -862,5 +893,6 @@ public class RouterConsoleRunner implements RouterApp {
             return rv;
         }
     }
+*****/
 
 }

history.txt

+2014-10-20 zzz
+ * Console: Fix lockups (ticket #1395)
+ * Eepsite Jetty: Switch back to QueuedThreadPool (ticket #1395)
+
 2014-10-17 zzz
  * NTCP: Deadlock fix 3rd try (ticket #1394)

installer/resources/eepsite/jetty.xml

@@ -53,26 +53,50 @@
 
       <!-- PICK ONE -->
 
-      <!-- If you don't have or want threadpool
-           Requests above the max will be queued
+      <!--
+           Recommended.
+           Two threads are used for the Connector and Acceptor.
+           Concurrent requests above maxThreads + queue size - 2 will be rejected and logged.
+           Due to the way QTP works, queue size should be larger than maxThreads.
+           Increase all values for high-traffic eepsites.
+
+           ref:
+           https://wiki.eclipse.org/Jetty/Howto/High_Load
+           http://trac.i2p2.i2p/ticket/1395
        -->
-     <!--
       <New class="org.eclipse.jetty.util.thread.QueuedThreadPool">
-        <Set name="minThreads">1</Set>
-        <Set name="maxThreads">16</Set>
+        <Arg>
+           <New class="java.util.concurrent.LinkedBlockingQueue">
+              <Arg type="int">40</Arg>
+           </New>
+        </Arg>
+        <Set name="minThreads">3</Set>
+        <Set name="maxThreads">20</Set>
+        <Set name="maxIdleTimeMs">60000</Set>
+        <Set name="daemon">true</Set>
+        <Set name="name">Eepsite Jetty</Set>
       </New>
-     -->
 
       <!-- Optional Java 5 bounded threadpool with job queue 
            Requests above the max will be rejected and logged.
            High-traffic sites should increase maximumPoolSize.
+
+           Args are:
+             corePoolSize (should be at least 3)
+             maximumPoolSize
+             keepAliveTime (milliseconds)
+             timeout (TimeUnit)
+             queue (BlockingQueue)
+
+           Not recommended.
+           ref:
+           http://trac.i2p2.i2p/ticket/1395
+           http://docs.oracle.com/javase/6/docs/api/java/util/concurrent/ThreadPoolExecutor.html
        -->
+     <!--
       <New class="org.eclipse.jetty.util.thread.ExecutorThreadPool">
-        <!-- corePoolSize (should be at least 3) -->
         <Arg type="int">3</Arg>
-        <!-- maximumPoolSize -->
         <Arg type="int">20</Arg>
-        <!-- keepAliveTime (milliseconds) -->
         <Arg type="long">60000</Arg>
         <Arg>
             <Call class="java.util.concurrent.TimeUnit" name="valueOf" >
@@ -83,6 +107,7 @@
             <New class="java.util.concurrent.SynchronousQueue" />
         </Arg>
       </New>
+     -->
     </Set>
 
 

router/java/src/net/i2p/router/RouterVersion.java

@@ -18,10 +18,10 @@ public class RouterVersion {
     /** deprecated */
     public final static String ID = "Monotone";
     public final static String VERSION = CoreVersion.VERSION;
-    public final static long BUILD = 13;
+    public final static long BUILD = 14;
 
     /** for example "-test" */
-    public final static String EXTRA = "";
+    public final static String EXTRA = "-rc";
     public final static String FULL_VERSION = VERSION + "-" + BUILD + EXTRA;
     public static void main(String args[]) {
         System.out.println("I2P Router version: " + FULL_VERSION);