* 2006-04-13 0.6.1.15 released

* Added a further failsafe against trying to queue up too many messages to
      a peer.

apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelIRCClient.java

@@ -27,6 +27,8 @@ public class I2PTunnelIRCClient extends I2PTunnelClientBase implements Runnable
     protected List dests;
     private static final long DEFAULT_READ_TIMEOUT = 5*60*1000; // -1
     protected long readTimeout = DEFAULT_READ_TIMEOUT;
+    /** this is the pong response the client expects for their last ping.  at least, i hope so... */
+    private String _expectedPong;
 
     /**
      * @throws IllegalArgumentException if the I2PTunnel does not contain
@@ -44,6 +46,8 @@ public class I2PTunnelIRCClient extends I2PTunnelClientBase implements Runnable
               l, 
               notifyThis, 
               "IRCHandler " + (++__clientId), tunnel);
+        
+        _expectedPong = null;
 
         StringTokenizer tok = new StringTokenizer(destinations, ",");
         dests = new ArrayList(1);
@@ -147,6 +151,8 @@ public class I2PTunnelIRCClient extends I2PTunnelClientBase implements Runnable
                             break;
                         if(inmsg.endsWith("\r"))
                             inmsg=inmsg.substring(0,inmsg.length()-1);
+                        if (_log.shouldLog(Log.DEBUG))
+                            _log.debug("in: [" + inmsg + "]");
                         String outmsg = inboundFilter(inmsg);
                         if(outmsg!=null)
                         {
@@ -217,6 +223,8 @@ public class I2PTunnelIRCClient extends I2PTunnelClientBase implements Runnable
                                 break;
                             if(inmsg.endsWith("\r"))
                                 inmsg=inmsg.substring(0,inmsg.length()-1);
+                            if (_log.shouldLog(Log.DEBUG))
+                                _log.debug("out: [" + inmsg + "]");
                             String outmsg = outboundFilter(inmsg);
                             if(outmsg!=null)
                             {
@@ -256,7 +264,7 @@ public class I2PTunnelIRCClient extends I2PTunnelClientBase implements Runnable
      *
      */
     
-    public static String inboundFilter(String s) {
+    public String inboundFilter(String s) {
         
         String field[]=s.split(" ",4);
         String command;
@@ -264,8 +272,8 @@ public class I2PTunnelIRCClient extends I2PTunnelClientBase implements Runnable
         final String[] allowedCommands =
         {
                 "NOTICE",
-                "PING",
-                "PONG",
+                //"PING",
+                //"PONG",
                 "MODE",
                 "JOIN",
                 "NICK",
@@ -293,6 +301,21 @@ public class I2PTunnelIRCClient extends I2PTunnelClientBase implements Runnable
             new Integer(command);
             return s;
         } catch(NumberFormatException nfe){}
+
+        
+	if ("PING".equals(command))
+            return "PING 127.0.0.1"; // no way to know what the ircd to i2ptunnel server con is, so localhost works
+	if ("PONG".equals(command)) {
+            // Turn the received ":irc.freshcoffee.i2p PONG irc.freshcoffee.i2p :127.0.0.1"
+            // into ":127.0.0.1 PONG 127.0.0.1 " so that the caller can append the client's extra parameter
+            // though, does 127.0.0.1 work for irc clients connecting remotely?  and for all of them?  sure would
+            // be great if irc clients actually followed the RFCs here, but i guess thats too much to ask.
+            // If we haven't PINGed them, or the PING we sent isn't something we know how to filter, this 
+            // is null.
+            String pong = _expectedPong;
+            _expectedPong = null;
+            return pong;
+        }
         
         // Allow all allowedCommands
         for(int i=0;i<allowedCommands.length;i++) {
@@ -324,14 +347,13 @@ public class I2PTunnelIRCClient extends I2PTunnelClientBase implements Runnable
         return null;
     }
     
-    public static String outboundFilter(String s) {
+    public String outboundFilter(String s) {
         
         String field[]=s.split(" ",3);
         String command;
         final String[] allowedCommands =
         {
                 "NOTICE",
-                "PONG",
                 "MODE",
                 "JOIN",
                 "NICK",
@@ -345,7 +367,8 @@ public class I2PTunnelIRCClient extends I2PTunnelClientBase implements Runnable
                 "MAP", // seems safe enough, the ircd should protect themselves though
                 "PART",
                 "OPER",
-                "PING",
+                // "PONG", // replaced with a filtered PING/PONG since some clients send the server IP (thanks aardvax!) 
+                // "PING", 
                 "KICK",
                 "HELPME",
                 "RULES",
@@ -361,6 +384,43 @@ public class I2PTunnelIRCClient extends I2PTunnelClientBase implements Runnable
         
         command = field[0].toUpperCase();
 
+	if ("PING".equals(command)) {
+            // Most clients just send a PING and are happy with any old PONG.  Others,
+            // like BitchX, actually expect certain behavior.  It sends two different pings:
+            // "PING :irc.freshcoffee.i2p" and "PING 1234567890 127.0.0.1" (where the IP is the proxy)
+            // the PONG to the former seems to be "PONG 127.0.0.1", while the PONG to the later is
+            // ":irc.freshcoffee.i2p PONG irc.freshcoffe.i2p :1234567890".
+            // We don't want to send them our proxy's IP address, so we need to rewrite the PING
+            // sent to the server, but when we get a PONG back, use what we expected, rather than
+            // what they sent.
+            //
+            // Yuck.
+
+            String rv = null;
+            if (field.length == 1) { // PING
+                rv = "PING";
+                _expectedPong = "PONG 127.0.0.1";
+            } else if (field.length == 2) { // PING nonce
+                rv = "PING " + field[1];
+                _expectedPong = "PONG " + field[1];
+            } else if (field.length == 3) { // PING nonce serverLocation
+                rv = "PING " + field[1];
+                _expectedPong = "PONG " + field[1];
+            } else {
+                if (_log.shouldLog(Log.ERROR))
+                    _log.error("IRC client sent a PING we don't understand, filtering it (\"" + s + "\")");
+                rv = null;
+                _expectedPong = null;
+            }
+            
+            if (_log.shouldLog(Log.WARN))
+                _log.warn("sending ping " + rv + ", waiting for " + _expectedPong + " orig was [" + s  + "]");
+            
+            return rv;
+        }
+	if ("PONG".equals(command))
+            return "PONG 127.0.0.1"; // no way to know what the ircd to i2ptunnel server con is, so localhost works
+
         // Allow all allowedCommands
         for(int i=0;i<allowedCommands.length;i++)
         {

apps/routerconsole/java/src/net/i2p/router/web/ConfigNetHandler.java

@@ -250,7 +250,7 @@ public class ConfigNetHandler extends FormHandler {
             // If hidden mode value changes, restart is required
             if (_hiddenMode && "false".equalsIgnoreCase(_context.getProperty(Router.PROP_HIDDEN, "false"))) {
                 _context.router().setConfigSetting(Router.PROP_HIDDEN, "true");
-                _context.router().getRouterInfo().addCapability(RouterInfo.CAPABILITY_HIDDEN);
+                _context.router().addCapabilities(_context.router().getRouterInfo());
                 addFormNotice("Gracefully restarting into Hidden Router Mode. Make sure you have no 0-1 length "
                               + "<a href=\"configtunnels.jsp\">tunnels!</a>");
                 hiddenSwitch();

apps/routerconsole/java/src/net/i2p/router/web/GraphHelper.java

@@ -55,6 +55,12 @@ public class GraphHelper {
     
     public String getImages() { 
         try {
+            _out.write("<img src=\"viewstat.jsp?stat=bw.combined"
+                           + "&amp;periodCount=" + _periodCount 
+                           + "&amp;width=" + _width
+                           + "&amp;height=" + _height
+                           + "\" title=\"Combined bandwidth graph\" />\n");
+            
             List listeners = StatSummarizer.instance().getListeners();
             for (int i = 0; i < listeners.size(); i++) {
                 SummaryListener lsnr = (SummaryListener)listeners.get(i);

apps/routerconsole/java/src/net/i2p/router/web/StatSummarizer.java

@@ -5,18 +5,27 @@ import java.util.*;
 
 import net.i2p.stat.*;
 import net.i2p.router.*;
+import net.i2p.util.Log;
+
+import java.awt.Color;
+import org.jrobin.graph.RrdGraph;
+import org.jrobin.graph.RrdGraphDef;
+import org.jrobin.graph.RrdGraphDefTemplate;
+import org.jrobin.core.RrdException;
 
 /**
  *
  */
 public class StatSummarizer implements Runnable {
     private RouterContext _context;
+    private Log _log;
     /** list of SummaryListener instances */
     private List _listeners;
     private static StatSummarizer _instance;
     
     public StatSummarizer() {
         _context = (RouterContext)RouterContext.listContexts().get(0); // fuck it, only summarize one per jvm
+        _log = _context.logManager().getLog(getClass());
         _listeners = new ArrayList(16);
         _instance = this;
     }
@@ -45,6 +54,10 @@ public class StatSummarizer implements Runnable {
                                                     ",router.activePeers.60000" +
                                                     ",router.activeSendPeers.60000" +
                                                     ",tunnel.acceptLoad.60000" +
+                                                    ",tunnel.dropLoadProactive.60000" +
+                                                    ",tunnel.buildExploratorySuccess.60000" +
+                                                    ",tunnel.buildExploratoryReject.60000" +
+                                                    ",tunnel.buildExploratoryExpire.60000" +
                                                     ",client.sendAckTime.60000" +
                                                     ",client.dispatchNoACK.60000" +
                                                     ",transport.sendMessageFailureLifetime.60000" +
@@ -124,6 +137,69 @@ public class StatSummarizer implements Runnable {
         return false;
     }
     
+    public boolean renderRatePng(OutputStream out, int width, int height, boolean hideLegend, boolean hideGrid, boolean hideTitle, boolean showEvents, int periodCount, boolean showCredit) throws IOException {
+        long end = _context.clock().now();
+        if (periodCount <= 0) periodCount = SummaryListener.PERIODS;
+        if (periodCount > SummaryListener.PERIODS)
+            periodCount = SummaryListener.PERIODS;
+        long period = 60*1000;
+        long start = end - period*periodCount;
+        long begin = System.currentTimeMillis();
+        try {
+            RrdGraphDef def = new RrdGraphDef();
+            def.setTimePeriod(start/1000, end/1000);
+            String title = "Bandwidth usage";
+            if (!hideTitle)
+                def.setTitle(title);
+            String sendName = SummaryListener.createName(_context, "bw.sendRate.60000");
+            String recvName = SummaryListener.createName(_context, "bw.recvRate.60000");
+            def.datasource(sendName, sendName, sendName, "AVERAGE", "MEMORY");
+            def.datasource(recvName, recvName, recvName, "AVERAGE", "MEMORY");
+            def.area(sendName, Color.BLUE, "Outbound bytes/second");
+            //def.line(sendName, Color.BLUE, "Outbound bytes/second", 3);
+            //def.line(recvName, Color.RED, "Inbound bytes/second@r", 3);
+            def.area(recvName, Color.RED, "Inbound bytes/second@r");
+            if (!hideLegend) {
+                def.gprint(sendName, "AVERAGE", "outbound average: @2@sbytes/second");
+                def.gprint(sendName, "MAX", " max: @2@sbytes/second@r");
+                def.gprint(recvName, "AVERAGE", "inbound average: @2bytes/second@s");
+                def.gprint(recvName, "MAX", " max: @2@sbytes/second@r");
+            }
+            if (!showCredit)
+                def.setShowSignature(false);
+            if (hideLegend) 
+                def.setShowLegend(false);
+            if (hideGrid) {
+                def.setGridX(false);
+                def.setGridY(false);
+            }
+            //System.out.println("rendering: path=" + path + " dsNames[0]=" + dsNames[0] + " dsNames[1]=" + dsNames[1] + " lsnr.getName=" + _listener.getName());
+            def.setAntiAliasing(false);
+            //System.out.println("Rendering: \n" + def.exportXmlTemplate());
+            //System.out.println("*****************\nData: \n" + _listener.getData().dump());
+            RrdGraph graph = new RrdGraph(def);
+            //System.out.println("Graph created");
+            byte data[] = null;
+            if ( (width <= 0) || (height <= 0) )
+                data = graph.getPNGBytes();
+            else
+                data = graph.getPNGBytes(width, height);
+            long timeToPlot = System.currentTimeMillis() - begin;
+            out.write(data);
+            //File t = File.createTempFile("jrobinData", ".xml");
+            //_listener.getData().dumpXml(new FileOutputStream(t));
+            //System.out.println("plotted: " + (data != null ? data.length : 0) + " bytes in " + timeToPlot
+            //                   ); // + ", data written to " + t.getAbsolutePath());
+            return true;
+        } catch (RrdException re) {
+            _log.error("Error rendering", re);
+            throw new IOException("Error plotting: " + re.getMessage());
+        } catch (IOException ioe) {
+            _log.error("Error rendering", ioe);
+            throw ioe;
+        }
+    }
+    
     /**
      * @param specs statName.period,statName.period,statName.period
      * @return list of Rate objects

apps/routerconsole/java/src/net/i2p/router/web/SummaryHelper.java

@@ -493,6 +493,13 @@ public class SummaryHelper {
         return _context.throttle().getTunnelLag() + "ms";
     }
     
+    public String getInboundBacklog() {
+        if (_context == null)
+            return "0";
+        
+        return String.valueOf(_context.tunnelManager().getInboundBuildQueueSize());
+    }
+    
     public boolean updateAvailable() { 
         return NewsFetcher.getInstance(_context).updateAvailable();
     }

apps/routerconsole/java/src/net/i2p/router/web/SummaryListener.java

@@ -78,7 +78,7 @@ class SummaryListener implements RateSummaryListener {
      * munged version from the user/developer-visible name.
      *
      */
-    private static String createName(I2PAppContext ctx, String wanted) { 
+    static String createName(I2PAppContext ctx, String wanted) { 
         return ctx.sha().calculateHash(DataHelper.getUTF8(wanted)).toBase64().substring(0,20);
     }
     

apps/routerconsole/jsp/summary.jsp

@@ -83,6 +83,7 @@
  <b>Job lag:</b> <jsp:getProperty name="helper" property="jobLag" /><br />
  <b>Message delay:</b> <jsp:getProperty name="helper" property="messageDelay" /><br />
  <b>Tunnel lag:</b> <jsp:getProperty name="helper" property="tunnelLag" /><br />
+ <b>Handle backlog:</b> <jsp:getProperty name="helper" property="inboundBacklog" /><br />
  <hr />
  
 </div>

apps/routerconsole/jsp/viewstat.jsp

@@ -9,18 +9,25 @@ if (templateFile != null) {
 net.i2p.stat.Rate rate = null;
 String stat = request.getParameter("stat");
 String period = request.getParameter("period");
+boolean fakeBw = (stat != null && ("bw.combined".equals(stat)));
 net.i2p.stat.RateStat rs = net.i2p.I2PAppContext.getGlobalContext().statManager().getRate(stat);
-if ( !rendered && (rs != null)) {
+if ( !rendered && ((rs != null) || fakeBw) ) {
   long per = -1;
   try { 
-    per = Long.parseLong(period); 
-    rate = rs.getRate(per);
-    if (rate != null) {
+    if (fakeBw)
+      per = 60*1000;
+    else
+      per = Long.parseLong(period); 
+    if (!fakeBw)
+      rate = rs.getRate(per);
+    if ( (rate != null) || (fakeBw) ) {
       java.io.OutputStream cout = response.getOutputStream();
       String format = request.getParameter("format");
       if ("xml".equals(format)) {
-        response.setContentType("text/xml");
-        rendered = net.i2p.router.web.StatSummarizer.instance().getXML(rate, cout);
+        if (!fakeBw) {
+          response.setContentType("text/xml");
+          rendered = net.i2p.router.web.StatSummarizer.instance().getXML(rate, cout);
+        }
       } else {
         response.setContentType("image/png");
         int width = -1;
@@ -39,7 +46,10 @@ if ( !rendered && (rs != null)) {
         boolean showCredit = true;
         if (request.getParameter("showCredit") != null)
           showCredit = Boolean.valueOf(""+request.getParameter("showCredit")).booleanValue();
-        rendered = net.i2p.router.web.StatSummarizer.instance().renderPng(rate, cout, width, height, hideLegend, hideGrid, hideTitle, showEvents, periodCount, showCredit);
+        if (fakeBw)
+            rendered = net.i2p.router.web.StatSummarizer.instance().renderRatePng(cout, width, height, hideLegend, hideGrid, hideTitle, showEvents, periodCount, showCredit);
+        else
+            rendered = net.i2p.router.web.StatSummarizer.instance().renderPng(rate, cout, width, height, hideLegend, hideGrid, hideTitle, showEvents, periodCount, showCredit);
       }
       if (rendered)
         cout.close();

apps/streaming/java/src/net/i2p/client/streaming/ConnectionOptions.java

@@ -101,7 +101,7 @@ public class ConnectionOptions extends I2PSocketOptionsImpl {
         setMaxWindowSize(getInt(opts, PROP_MAX_WINDOW_SIZE, Connection.MAX_WINDOW_SIZE));
         setConnectDelay(getInt(opts, PROP_CONNECT_DELAY, -1));
         setProfile(getInt(opts, PROP_PROFILE, PROFILE_BULK));
-        setMaxMessageSize(getInt(opts, PROP_MAX_MESSAGE_SIZE, 4*1024));
+        setMaxMessageSize(getInt(opts, PROP_MAX_MESSAGE_SIZE, 960)); // 960 fits inside a single tunnel message
         setRTT(getInt(opts, PROP_INITIAL_RTT, 10*1000));
         setReceiveWindow(getInt(opts, PROP_INITIAL_RECEIVE_WINDOW, 1));
         setResendDelay(getInt(opts, PROP_INITIAL_RESEND_DELAY, 1000));

apps/syndie/java/src/net/i2p/syndie/Archive.java

@@ -28,6 +28,7 @@ import net.i2p.util.Log;
 public class Archive {
     private I2PAppContext _context;
     private Log _log;
+    private BlogManager _mgr;
     private File _rootDir;
     private File _cacheDir;
     private Map _blogInfo;
@@ -43,9 +44,10 @@ public class Archive {
         public boolean accept(File dir, String name) { return name.endsWith(".snd"); }
     };    
     
-    public Archive(I2PAppContext ctx, String rootDir, String cacheDir) {
+    public Archive(I2PAppContext ctx, String rootDir, String cacheDir, BlogManager mgr) {
         _context = ctx;
         _log = ctx.logManager().getLog(Archive.class);
+        _mgr = mgr;
         _rootDir = new File(rootDir);
         if (!_rootDir.exists())
             _rootDir.mkdirs();
@@ -72,6 +74,13 @@ public class Archive {
                     try {
                         fi = new FileInputStream(meta);
                         bi.load(fi);
+                        if (_mgr.isBanned(bi.getKey().calculateHash())) {
+                            fi.close();
+                            fi = null;
+                            _log.error("Deleting banned blog " + bi.getKey().calculateHash().toBase64());
+                            delete(bi.getKey().calculateHash());
+                            continue;
+                        }
                         if (bi.verify(_context)) {
                             info.add(bi);
                         } else {
@@ -120,6 +129,12 @@ public class Archive {
             _log.warn("Not storing invalid blog " + info);
             return false;
         }
+        
+        if (_mgr.isBanned(info.getKey().calculateHash())) {
+            _log.error("Not storing banned blog " + info.getKey().calculateHash().toBase64(), new Exception("Stored by"));
+            return false;
+        }
+        
         boolean isNew = true;
         synchronized (_blogInfo) {
             BlogInfo old = (BlogInfo)_blogInfo.get(info.getKey().calculateHash());
@@ -274,8 +289,9 @@ public class Archive {
                 if (blogKey == null) {
                     // no key, cache.
                     File entryDir = getEntryDir(entries[i]);
-                    if (entryDir.exists())
+                    if (entryDir.exists()) {
                         entry = getCachedEntry(entryDir);
+                    }
                     if ((entry == null) || !entryDir.exists()) {
                         if (!extractEntry(entries[i], entryDir, info)) {
                             _log.error("Entry " + entries[i].getPath() + " is not valid");

apps/syndie/java/src/net/i2p/syndie/BlogManager.java

@@ -74,7 +74,7 @@ public class BlogManager {
         _cacheDir.mkdirs();
         _userDir.mkdirs();
         _tempDir.mkdirs();
-        _archive = new Archive(ctx, _archiveDir.getAbsolutePath(), _cacheDir.getAbsolutePath());
+        _archive = new Archive(ctx, _archiveDir.getAbsolutePath(), _cacheDir.getAbsolutePath(), this);
         if (regenIndex)
             _archive.regenerateIndex();
     }
@@ -890,6 +890,8 @@ public class BlogManager {
         try {
             BlogInfo info = new BlogInfo();
             info.load(metadataStream);
+            if (isBanned(info.getKey().calculateHash()))
+                return false;
             return _archive.storeBlogInfo(info);
         } catch (IOException ioe) {
             _log.error("Error importing meta", ioe);
@@ -906,6 +908,8 @@ public class BlogManager {
         try {
             EntryContainer c = new EntryContainer();
             c.load(entryStream);
+            if (isBanned(c.getURI().getKeyHash()))
+                return false;
             return _archive.storeEntry(c);
         } catch (IOException ioe) {
             _log.error("Error importing entry", ioe);
@@ -1062,7 +1066,7 @@ public class BlogManager {
     }
     
     public boolean isBanned(Hash blog) {
-        if (blog == null) return false;
+        if ( (blog == null) || (blog.getData() == null) || (blog.getData().length <= 0) ) return false;
         String str = blog.toBase64();
         String banned = System.getProperty("syndie.bannedBlogs", "");
         return (banned.indexOf(str) >= 0);

apps/syndie/java/src/net/i2p/syndie/data/ArchiveIndex.java

@@ -163,8 +163,9 @@ public class ArchiveIndex {
     /** list of unique blogs locally known (set of Hash) */
     public Set getUniqueBlogs() {
         Set rv = new HashSet();
-        for (int i = 0; i < _blogs.size(); i++)
+        for (int i = 0; i < _blogs.size(); i++) {
             rv.add(getBlog(i));
+        }
         return rv;
     }
     public List getReplies(BlogURI uri) {
@@ -367,7 +368,10 @@ public class ArchiveIndex {
             return;
         tok.nextToken();
         String keyStr = tok.nextToken();
-        Hash keyHash = new Hash(Base64.decode(keyStr));
+        byte k[] = Base64.decode(keyStr);
+        if ( (k == null) || (k.length != Hash.HASH_LENGTH) )
+            return; // ignore bad hashes
+        Hash keyHash = new Hash(k);
         String whenStr = tok.nextToken();
         long when = getIndexDate(whenStr);
         String tag = tok.nextToken();

apps/syndie/java/src/net/i2p/syndie/web/BaseServlet.java

@@ -295,7 +295,7 @@ public abstract class BaseServlet extends HttpServlet {
         
         if (AddressesServlet.ACTION_ADD_TAG.equals(action)) {
             String name = req.getParameter(AddressesServlet.PARAM_NAME);
-            if (!user.getPetNameDB().containsName(name)) {
+            if ((name != null) && (name.trim().length() > 0) && (!user.getPetNameDB().containsName(name)) ) {
                 PetName pn = new PetName(name, AddressesServlet.NET_SYNDIE, AddressesServlet.PROTO_TAG, name);
                 user.getPetNameDB().add(pn);
                 BlogManager.instance().saveUser(user);
@@ -307,7 +307,7 @@ public abstract class BaseServlet extends HttpServlet {
              (AddressesServlet.ACTION_ADD_OTHER.equals(action)) ||
              (AddressesServlet.ACTION_ADD_PEER.equals(action)) ) {
             PetName pn = buildNewAddress(req);
-            if ( (pn != null) && (pn.getName() != null) && (pn.getLocation() != null) && 
+            if ( (pn != null) && (pn.getName() != null) && (pn.getName().trim().length() > 0) && (pn.getLocation() != null) && 
                  (!user.getPetNameDB().containsName(pn.getName())) ) {
                 user.getPetNameDB().add(pn);
                 BlogManager.instance().saveUser(user);
@@ -744,6 +744,8 @@ public abstract class BaseServlet extends HttpServlet {
         for (Iterator iter = names.iterator(); iter.hasNext(); ) {
             String name = (String) iter.next();
             PetName pn = db.getByName(name);
+            if (pn == null)
+                continue;
             String proto = pn.getProtocol();
             String loc = pn.getLocation();
             if (proto != null && loc != null && "syndieblog".equals(proto) && pn.isMember(FilteredThreadIndex.GROUP_FAVORITE)) {

apps/syndie/java/src/net/i2p/syndie/web/RemoteArchiveBean.java

@@ -641,6 +641,8 @@ public class RemoteArchiveBean {
         int newBlogs = 0;
         for (Iterator iter = remoteBlogs.iterator(); iter.hasNext(); ) {
             Hash blog = (Hash)iter.next();
+            if ( (blog == null) || (blog.getData() == null) || (blog.getData().length <= 0) )
+                continue;
             if (ignoreBlog(user, blog))
                 continue;
             if (!localBlogs.contains(blog)) {

apps/syndie/jsp/syndie.css

@@ -134,9 +134,9 @@ select {
    display: inline;
 }
 .controlBar {
-	margin: 0em;
-	padding: 0em;
-//	border: medium solid #DDF;
+        border-bottom: thick double #CCF;
+        border-left: medium solid #CCF;
+        border-right: medium solid #CCF;
 	background-color: #EEF;
 	color: inherit;
 	font-size: small;

core/java/src/net/i2p/CoreVersion.java

@@ -14,8 +14,8 @@ package net.i2p;
  *
  */
 public class CoreVersion {
-    public final static String ID = "$Revision: 1.55 $ $Date: 2006/02/27 14:05:41 $";
-    public final static String VERSION = "0.6.1.13";
+    public final static String ID = "$Revision: 1.57 $ $Date: 2006/04/05 12:08:07 $";
+    public final static String VERSION = "0.6.1.15";
 
     public static void main(String args[]) {
         System.out.println("I2P Core version: " + VERSION);

core/java/src/net/i2p/stat/Rate.java

@@ -188,6 +188,8 @@ public class Rate {
             long measuredPeriod = now - _lastCoalesceDate;
             if (measuredPeriod < _period - SLACK) {
                 // no need to coalesce (assuming we only try to do so once per minute)
+                if (_log.shouldLog(Log.WARN))
+                    _log.warn("not coalescing, measuredPeriod = " + measuredPeriod + " period = " + _period);
                 return;
             }
     

history.txt

@@ -1,4 +1,91 @@
-$Id: history.txt,v 1.439 2006/03/25 18:50:51 jrandom Exp $
+$Id: history.txt,v 1.454 2006/04/12 23:22:07 jrandom Exp $
+
+* 2006-04-13  0.6.1.15 released
+
+2006-04-12  jrandom
+    * Added a further failsafe against trying to queue up too many messages to
+      a peer.
+
+2006-04-12  jrandom
+    * Watch out for failed syndie index fetches (thanks bar!)
+
+2006-04-11  jrandom
+    * Throttling improvements on SSU - throttle all transmissions to a peer
+      when we are retransmitting, not just retransmissions.  Also, if
+      we're already retransmitting to a peer, probabalistically tail drop new
+      messages targetting that peer, based on the estimated wait time before
+      transmission.
+    * Fixed the rounding error in the inbound tunnel drop probability.
+
+2006-04-10  jrandom
+    * Include a combined send/receive graph (good idea cervantes!)
+    * Proactively drop inbound tunnel requests probabalistically as the
+      estimated queue time approaches our limit, rather than letting them all
+      through up to that limit.
+
+2006-04-08  jrandom
+    * Stat summarization fix (removing the occational holes in the jrobin
+      graphs)
+
+2006-04-08  jrandom
+    * Process inbound tunnel requests more efficiently
+    * Proactively drop inbound tunnel requests if the queue before we'd
+      process it in is too long (dynamically adjusted by cpu load)
+    * Adjust the tunnel rejection throttle to reject requeusts when we have to
+      proactively drop too many requests.
+    * Display the number of pending inbound tunnel join requests on the router
+      console (as the "handle backlog")
+    * Include a few more stats in the default set of graphs
+
+2006-04-06  jrandom
+    * Fix for a bug in the new irc ping/pong filter (thanks Complication!)
+
+2006-04-06  jrandom
+    * Fixed a typo in the reply cleanup code
+
+* 2006-04-05  0.6.1.14 released
+
+2006-04-05  jrandom
+    * Cut down on the time that we allow a tunnel creation request to sit by
+      without response, and reject tunnel creation requests that are lagged
+      locally.  Also switch to a bounded FIFO instead of a LIFO
+    * Threading tweaks for the message handling (thanks bar!)
+    * Don't add addresses to syndie with blank names (thanks Complication!)
+    * Further ban clearance
+
+2006-04-05  jrandom
+    * Fix during the ssu handshake to avoid an unnecessary failure on
+      packet retransmission (thanks ripple!)
+    * Fix during the SSU handshake to use the negotiated session key asap,
+      rather than using the intro key for more than we should (thanks ripple!)
+    * Fixes to the message reply registry (thanks Complication!)
+    * More comprehensive syndie banning (for repeated pushes)
+    * Publish the router's ballpark bandwidth limit (w/in a power of 2), for 
+      testing purposes
+    * Put a floor back on the capacity threshold, so too many failing peers
+      won't cause us to pick very bad peers (unless we have very few good
+      ones)
+    * Bugfix to cut down on peers using introducers unneessarily (thanks
+      Complication!)
+    * Reduced the default streaming lib message size to fit into a single
+      tunnel message, rather than require 5 tunnel messages to be transferred
+      without loss before recomposition.  This reduces throughput, but should
+      increase reliability, at least for the time being.
+    * Misc small bugfixes in the router (thanks all!)
+    * More tweaking for Syndie's CSS (thanks Doubtful Salmon!)
+
+2006-04-01  jrandom
+    * Take out the router watchdog's teeth (don't restart on leaseset failure)
+    * Filter the IRC ping/pong messages, as some clients send unsafe
+      information in them (thanks aardvax and dust!)
+
+2006-03-30  jrandom
+    * Substantially reduced the lock contention in the message registry (a
+      major hotspot that can choke most threads).  Also reworked the locking
+      so we don't need per-message timer events
+    * No need to have additional per-peer message clearing, as they are
+      either unregistered individually or expired.
+    * Include some of the more transient tunnel throttling
 
 * 2006-03-26  0.6.1.13 released
 

initialNews.xml

@@ -1,5 +1,5 @@
-<i2p.news date="$Date: 2006/02/27 14:05:41 $">
- <i2p.release version="0.6.1.13" date="2006/03/26" minVersion="0.6"
+<i2p.news date="$Date: 2006/04/05 12:08:05 $">
+ <i2p.release version="0.6.1.15" date="2006/04/13" minVersion="0.6"
               anonurl="http://i2p/NF2RLVUxVulR3IqK0sGJR0dHQcGXAzwa6rEO4WAWYXOHw-DoZhKnlbf1nzHXwMEJoex5nFTyiNMqxJMWlY54cvU~UenZdkyQQeUSBZXyuSweflUXFqKN-y8xIoK2w9Ylq1k8IcrAFDsITyOzjUKoOPfVq34rKNDo7fYyis4kT5bAHy~2N1EVMs34pi2RFabATIOBk38Qhab57Umpa6yEoE~rbyR~suDRvD7gjBvBiIKFqhFueXsR2uSrPB-yzwAGofTXuklofK3DdKspciclTVzqbDjsk5UXfu2nTrC1agkhLyqlOfjhyqC~t1IXm-Vs2o7911k7KKLGjB4lmH508YJ7G9fLAUyjuB-wwwhejoWqvg7oWvqo4oIok8LG6ECR71C3dzCvIjY2QcrhoaazA9G4zcGMm6NKND-H4XY6tUWhpB~5GefB3YczOqMbHq4wi0O9MzBFrOJEOs3X4hwboKWANf7DT5PZKJZ5KorQPsYRSq0E3wSOsFCSsdVCKUGsAAAA/i2p/i2pupdate.sud"
               publicurl="http://dev.i2p.net/i2p/i2pupdate.sud"
               anonannouncement="http://i2p/NF2RLVUxVulR3IqK0sGJR0dHQcGXAzwa6rEO4WAWYXOHw-DoZhKnlbf1nzHXwMEJoex5nFTyiNMqxJMWlY54cvU~UenZdkyQQeUSBZXyuSweflUXFqKN-y8xIoK2w9Ylq1k8IcrAFDsITyOzjUKoOPfVq34rKNDo7fYyis4kT5bAHy~2N1EVMs34pi2RFabATIOBk38Qhab57Umpa6yEoE~rbyR~suDRvD7gjBvBiIKFqhFueXsR2uSrPB-yzwAGofTXuklofK3DdKspciclTVzqbDjsk5UXfu2nTrC1agkhLyqlOfjhyqC~t1IXm-Vs2o7911k7KKLGjB4lmH508YJ7G9fLAUyjuB-wwwhejoWqvg7oWvqo4oIok8LG6ECR71C3dzCvIjY2QcrhoaazA9G4zcGMm6NKND-H4XY6tUWhpB~5GefB3YczOqMbHq4wi0O9MzBFrOJEOs3X4hwboKWANf7DT5PZKJZ5KorQPsYRSq0E3wSOsFCSsdVCKUGsAAAA/pipermail/i2p/2005-September/000878.html" 

installer/install.xml

@@ -4,7 +4,7 @@
 
     <info>
         <appname>i2p</appname>
-        <appversion>0.6.1.13</appversion>
+        <appversion>0.6.1.15</appversion>
         <authors>
             <author name="I2P" email="support@i2p.net"/>
         </authors>

news.xml

@@ -1,5 +1,5 @@
-<i2p.news date="$Date: 2006/03/21 21:15:13 $">
- <i2p.release version="0.6.1.13" date="2006/03/26" minVersion="0.6"
+<i2p.news date="$Date: 2006/04/05 12:08:04 $">
+ <i2p.release version="0.6.1.15" date="2006/04/13" minVersion="0.6"
               anonurl="http://i2p/NF2RLVUxVulR3IqK0sGJR0dHQcGXAzwa6rEO4WAWYXOHw-DoZhKnlbf1nzHXwMEJoex5nFTyiNMqxJMWlY54cvU~UenZdkyQQeUSBZXyuSweflUXFqKN-y8xIoK2w9Ylq1k8IcrAFDsITyOzjUKoOPfVq34rKNDo7fYyis4kT5bAHy~2N1EVMs34pi2RFabATIOBk38Qhab57Umpa6yEoE~rbyR~suDRvD7gjBvBiIKFqhFueXsR2uSrPB-yzwAGofTXuklofK3DdKspciclTVzqbDjsk5UXfu2nTrC1agkhLyqlOfjhyqC~t1IXm-Vs2o7911k7KKLGjB4lmH508YJ7G9fLAUyjuB-wwwhejoWqvg7oWvqo4oIok8LG6ECR71C3dzCvIjY2QcrhoaazA9G4zcGMm6NKND-H4XY6tUWhpB~5GefB3YczOqMbHq4wi0O9MzBFrOJEOs3X4hwboKWANf7DT5PZKJZ5KorQPsYRSq0E3wSOsFCSsdVCKUGsAAAA/i2p/i2pupdate.sud"
               publicurl="http://dev.i2p.net/i2p/i2pupdate.sud"
               anonannouncement="http://i2p/NF2RLVUxVulR3IqK0sGJR0dHQcGXAzwa6rEO4WAWYXOHw-DoZhKnlbf1nzHXwMEJoex5nFTyiNMqxJMWlY54cvU~UenZdkyQQeUSBZXyuSweflUXFqKN-y8xIoK2w9Ylq1k8IcrAFDsITyOzjUKoOPfVq34rKNDo7fYyis4kT5bAHy~2N1EVMs34pi2RFabATIOBk38Qhab57Umpa6yEoE~rbyR~suDRvD7gjBvBiIKFqhFueXsR2uSrPB-yzwAGofTXuklofK3DdKspciclTVzqbDjsk5UXfu2nTrC1agkhLyqlOfjhyqC~t1IXm-Vs2o7911k7KKLGjB4lmH508YJ7G9fLAUyjuB-wwwhejoWqvg7oWvqo4oIok8LG6ECR71C3dzCvIjY2QcrhoaazA9G4zcGMm6NKND-H4XY6tUWhpB~5GefB3YczOqMbHq4wi0O9MzBFrOJEOs3X4hwboKWANf7DT5PZKJZ5KorQPsYRSq0E3wSOsFCSsdVCKUGsAAAA/pipermail/i2p/2005-September/000878.html" 
@@ -10,14 +10,12 @@
             anonlogs="http://i2p/Nf3ab-ZFkmI-LyMt7GjgT-jfvZ3zKDl0L96pmGQXF1B82W2Bfjf0n7~288vafocjFLnQnVcmZd~-p0-Oolfo9aW2Rm-AhyqxnxyLlPBqGxsJBXjPhm1JBT4Ia8FB-VXt0BuY0fMKdAfWwN61-tj4zIcQWRxv3DFquwEf035K~Ra4SWOqiuJgTRJu7~o~DzHVljVgWIzwf8Z84cz0X33pv-mdG~~y0Bsc2qJVnYwjjR178YMcRSmNE0FVMcs6f17c6zqhMw-11qjKpY~EJfHYCx4lBWF37CD0obbWqTNUIbL~78vxqZRT3dgAgnLixog9nqTO-0Rh~NpVUZnoUi7fNR~awW5U3Cf7rU7nNEKKobLue78hjvRcWn7upHUF45QqTDuaM3yZa7OsjbcH-I909DOub2Q0Dno6vIwuA7yrysccN1sbnkwZbKlf4T6~iDdhaSLJd97QCyPOlbyUfYy9QLNExlRqKgNVJcMJRrIual~Lb1CLbnzt0uvobM57UpqSAAAA/meeting141"
             publiclogs="http://www.i2p.net/meeting141" />
 &#149;
-2006-02-27:
-<a href="http://dev.i2p/pipermail/i2p/2006-February/001265.html">0.6.1.12 released</a>
-with some useful SSU and peer selection fixes - please upgrade as soon as possible.
+2006-04-13: 0.6.1.15 released with helpful tunnel building and ssu improvements.
 <br>
 &#149;
-2006-03-21:
-<a href="http://dev.i2p/pipermail/i2p/2006-March/001271.html">status notes</a>
+2006-04-04:
+<a href="http://dev.i2p/pipermail/i2p/2006-April/001275.html">status notes</a>
 and
-<a href="http://www.i2p/meeting173">meeting log</a>
+<a href="http://www.i2p/meeting175">meeting log</a>
 <br>
 </i2p.news>

router/java/src/net/i2p/data/i2np/DatabaseLookupMessage.java

@@ -44,9 +44,9 @@ public class DatabaseLookupMessage extends I2NPMessageImpl {
     }
     public DatabaseLookupMessage(I2PAppContext context, boolean locallyCreated) {
         super(context);
-        setSearchKey(null);
-        setFrom(null);
-        setDontIncludePeers(null);
+        //setSearchKey(null);
+        //setFrom(null);
+        //setDontIncludePeers(null);
         
         context.statManager().createRateStat("router.throttleNetDbDoSSend", "How many netDb lookup messages we are sending during a period with a DoS detected", "Throttle", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
     

router/java/src/net/i2p/data/i2np/I2NPMessageImpl.java

@@ -290,7 +290,11 @@ public abstract class I2NPMessageImpl extends DataStructureImpl implements I2NPM
 
     public void readMessage(byte data[], int offset, int dataSize, int type, I2NPMessageHandler handler) throws I2NPMessageException, IOException {
         // ignore the handler (overridden in subclasses if necessary
-        readMessage(data, offset, dataSize, type);
+        try {
+            readMessage(data, offset, dataSize, type);
+        } catch (IllegalArgumentException iae) {
+            throw new I2NPMessageException("Error reading the message", iae);
+        }
     }
 
     
@@ -313,20 +317,24 @@ public abstract class I2NPMessageImpl extends DataStructureImpl implements I2NPM
             return msg;
         }
 
-        long expiration = DataHelper.fromLong(buffer, offset, 4) * 1000; // seconds
-        offset += 4;
-        int dataSize = len - 1 - 4;
         try {
+            long expiration = DataHelper.fromLong(buffer, offset, 4) * 1000; // seconds
+            offset += 4;
+            int dataSize = len - 1 - 4;
             msg.readMessage(buffer, offset, dataSize, type, handler);
             msg.setMessageExpiration(expiration);
             msg.read();
             return msg;
         } catch (IOException ioe) {
             throw new I2NPMessageException("IO error reading raw message", ioe);
+        } catch (IllegalArgumentException iae) {
+            throw new I2NPMessageException("Corrupt message (negative expiration)", iae);
         }
     }
 
-    protected void verifyUnwritten() { if (_written) throw new RuntimeException("Already written"); }
+    protected void verifyUnwritten() { 
+        if (_written) throw new RuntimeException("Already written"); 
+    }
     protected void written() { _written = true; }
     protected void read() { _read = true; }
     

router/java/src/net/i2p/router/Router.java

@@ -323,13 +323,8 @@ public class Router {
             stats.setProperty(RouterInfo.PROP_NETWORK_ID, NETWORK_ID+"");
             ri.setOptions(stats);
             ri.setAddresses(_context.commSystem().createAddresses());
-            if (FloodfillNetworkDatabaseFacade.floodfillEnabled(_context))
-                ri.addCapability(FloodfillNetworkDatabaseFacade.CAPACITY_FLOODFILL);
-            if("true".equalsIgnoreCase(_context.getProperty(Router.PROP_HIDDEN, "false"))) {
-                ri.addCapability(RouterInfo.CAPABILITY_HIDDEN);
-            }
 
-            addReachabilityCapability(ri);
+            addCapabilities(ri);
             SigningPrivateKey key = _context.keyManager().getSigningPrivateKey();
             if (key == null) {
                 _log.log(Log.CRIT, "Internal error - signing private key not known?  wtf");
@@ -358,15 +353,43 @@ public class Router {
         }
     }
     
+    // publicize our ballpark capacity - this does not affect anything at
+    // the moment
+    public static final char CAPABILITY_BW16 = 'K';
+    public static final char CAPABILITY_BW32 = 'L';
+    public static final char CAPABILITY_BW64 = 'M';
+    public static final char CAPABILITY_BW128 = 'N';
+    public static final char CAPABILITY_BW256 = 'O';
+    
     public static final char CAPABILITY_REACHABLE = 'R';
     public static final char CAPABILITY_UNREACHABLE = 'U';
     public static final String PROP_FORCE_UNREACHABLE = "router.forceUnreachable";
 
     public static final char CAPABILITY_NEW_TUNNEL = 'T';
     
-    public void addReachabilityCapability(RouterInfo ri) {
-        // routers who can understand TunnelBuildMessages 
-        ////ri.addCapability(CAPABILITY_NEW_TUNNEL);
+    public void addCapabilities(RouterInfo ri) {
+        int bwLim = Math.min(_context.bandwidthLimiter().getInboundKBytesPerSecond(),
+                             _context.bandwidthLimiter().getInboundKBytesPerSecond());
+        if (_log.shouldLog(Log.WARN))
+            _log.warn("Adding capabilities w/ bw limit @ " + bwLim, new Exception("caps"));
+        
+        if (bwLim <= 16) {
+            ri.addCapability(CAPABILITY_BW16);
+        } else if (bwLim <= 32) {
+            ri.addCapability(CAPABILITY_BW32);
+        } else if (bwLim <= 64) {
+            ri.addCapability(CAPABILITY_BW64);
+        } else if (bwLim <= 128) {
+            ri.addCapability(CAPABILITY_BW128);
+        } else { // ok, more than 128KBps... aka "lots"
+            ri.addCapability(CAPABILITY_BW256);
+        }
+        
+        if (FloodfillNetworkDatabaseFacade.floodfillEnabled(_context))
+            ri.addCapability(FloodfillNetworkDatabaseFacade.CAPACITY_FLOODFILL);
+        
+        if("true".equalsIgnoreCase(_context.getProperty(Router.PROP_HIDDEN, "false")))
+            ri.addCapability(RouterInfo.CAPABILITY_HIDDEN);
         
         String forceUnreachable = _context.getProperty(PROP_FORCE_UNREACHABLE);
         if ( (forceUnreachable != null) && ("true".equalsIgnoreCase(forceUnreachable)) ) {
@@ -1064,7 +1087,7 @@ class CoalesceStatsEvent implements SimpleTimer.TimedEvent {
             }
         }
         
-        SimpleTimer.getInstance().addEvent(this, 60*1000);
+        SimpleTimer.getInstance().addEvent(this, 20*1000);
     }
 }
 

router/java/src/net/i2p/router/RouterThrottleImpl.java

@@ -248,7 +248,7 @@ class RouterThrottleImpl implements RouterThrottle {
      */
     private boolean allowTunnel(double bytesAllocated, int numTunnels) {
         int maxKBps = Math.min(_context.bandwidthLimiter().getOutboundKBytesPerSecond(), _context.bandwidthLimiter().getInboundKBytesPerSecond());
-        int used1s = 0; //get1sRate(_context); // dont throttle on the 1s rate, its too volatile
+        int used1s = get1sRate(_context); // dont throttle on the 1s rate, its too volatile
         int used1m = get1mRate(_context);
         int used5m = 0; //get5mRate(_context); // don't throttle on the 5m rate, as that'd hide available bandwidth
         int used = Math.max(Math.max(used1s, used1m), used5m);

router/java/src/net/i2p/router/RouterVersion.java

@@ -15,8 +15,8 @@ import net.i2p.CoreVersion;
  *
  */
 public class RouterVersion {
-    public final static String ID = "$Revision: 1.380 $ $Date: 2006/03/25 18:50:48 $";
-    public final static String VERSION = "0.6.1.13";
+    public final static String ID = "$Revision: 1.394 $ $Date: 2006/04/12 23:22:33 $";
+    public final static String VERSION = "0.6.1.15";
     public final static long BUILD = 0;
     public static void main(String args[]) {
         System.out.println("I2P Router version: " + VERSION + "-" + BUILD);

router/java/src/net/i2p/router/RouterWatchdog.java

@@ -47,12 +47,13 @@ class RouterWatchdog implements Runnable {
     }
     
     private boolean shutdownOnHang() {
-        return Boolean.valueOf(_context.getProperty("watchdog.haltOnHang", "true")).booleanValue();
+        return Boolean.valueOf(_context.getProperty("watchdog.haltOnHang", "false")).booleanValue();
     }
     
     private void dumpStatus() {
         if (_log.shouldLog(Log.ERROR)) {
             Job cur = _context.jobQueue().getLastJob();
+            /*
             if (cur != null) 
                 _log.error("Most recent job: " + cur);
             _log.error("Last job began: " 
@@ -61,6 +62,7 @@ class RouterWatchdog implements Runnable {
             _log.error("Last job ended: " 
                        + DataHelper.formatDuration(_context.clock().now()-_context.jobQueue().getLastJobEnd())
                        + " ago");
+            */
             _log.error("Ready and waiting jobs: " + _context.jobQueue().getReadyCount());
             _log.error("Job lag: " + _context.jobQueue().getMaxLag());
             _log.error("Participating tunnel count: " + _context.tunnelManager().getParticipatingCount());

router/java/src/net/i2p/router/Shitlist.java

@@ -84,7 +84,7 @@ public class Shitlist {
         
         _context.netDb().fail(peer);
         //_context.tunnelManager().peerFailed(peer);
-        _context.messageRegistry().peerFailed(peer);
+        //_context.messageRegistry().peerFailed(peer);
         if (!wasAlready)
             _context.messageHistory().shitlist(peer, reason);
         return wasAlready;

router/java/src/net/i2p/router/TunnelManagerFacade.java

@@ -56,6 +56,9 @@ public interface TunnelManagerFacade extends Service {
     /** When does the last tunnel we are participating in expire? */
     public long getLastParticipatingExpiration();
     
+    /** count how many inbound tunnel requests we have received but not yet processed */
+    public int getInboundBuildQueueSize();
+    
     /** 
      * the client connected (or updated their settings), so make sure we have
      * the tunnels for them, and whenever necessary, ask them to authorize 
@@ -97,6 +100,7 @@ class DummyTunnelManagerFacade implements TunnelManagerFacade {
     public void setOutboundSettings(TunnelPoolSettings settings) {}
     public void setInboundSettings(Hash client, TunnelPoolSettings settings) {}
     public void setOutboundSettings(Hash client, TunnelPoolSettings settings) {}
+    public int getInboundBuildQueueSize() { return 0; }
     
     public void renderStatusHTML(Writer out) throws IOException {}
     public void restart() {}

router/java/src/net/i2p/router/message/GarlicMessageBuilder.java

@@ -123,7 +123,8 @@ public class GarlicMessageBuilder {
         
         long timeFromNow = config.getExpiration() - ctx.clock().now();
         if (timeFromNow < 1*1000) {
-            log.error("Building a message expiring in " + timeFromNow + "ms: " + config, new Exception("created by"));
+            if (log.shouldLog(Log.WARN))
+                log.warn("Building a message expiring in " + timeFromNow + "ms: " + config, new Exception("created by"));
             return null;
         }
         

router/java/src/net/i2p/router/message/OutboundClientMessageJobHelper.java

@@ -131,9 +131,9 @@ class OutboundClientMessageJobHelper {
         TunnelId replyToTunnelId = null; // tunnel id on that gateway
         
         if (replyToTunnel == null) {
-            if (log.shouldLog(Log.ERROR))
-                log.error("Unable to send client message from " + from.toBase64() 
-                          + ", as there are no inbound tunnels available");
+            if (log.shouldLog(Log.WARN))
+                log.warn("Unable to send client message from " + from.toBase64() 
+                         + ", as there are no inbound tunnels available");
             return null;
         }
         replyToTunnelId = replyToTunnel.getReceiveTunnelId(0);

router/java/src/net/i2p/router/message/OutboundClientMessageOneShotJob.java

@@ -213,8 +213,8 @@ public class OutboundClientMessageOneShotJob extends JobImpl {
             if (ok) {
                 send();
             } else {
-                if (_log.shouldLog(Log.ERROR))
-                    _log.error("Unable to send on a random lease, as getNext returned null (to=" + _toString + ")");
+                if (_log.shouldLog(Log.WARN))
+                    _log.warn("Unable to send on a random lease, as getNext returned null (to=" + _toString + ")");
                 dieFatal();
             }
         }

router/java/src/net/i2p/router/networkdb/PublishLocalRouterInfoJob.java

@@ -45,13 +45,8 @@ public class PublishLocalRouterInfoJob extends JobImpl {
             ri.setPublished(getContext().clock().now());
             ri.setOptions(stats);
             ri.setAddresses(getContext().commSystem().createAddresses());
-            if (FloodfillNetworkDatabaseFacade.floodfillEnabled(getContext()))
-                ri.addCapability(FloodfillNetworkDatabaseFacade.CAPACITY_FLOODFILL);
 
-            if ("true".equalsIgnoreCase(getContext().getProperty(Router.PROP_HIDDEN, "false")))
-                ri.addCapability(RouterInfo.CAPABILITY_HIDDEN);
-
-            getContext().router().addReachabilityCapability(ri);
+            getContext().router().addCapabilities(ri);
             SigningPrivateKey key = getContext().keyManager().getSigningPrivateKey();
             if (key == null) {
                 _log.log(Log.CRIT, "Internal error - signing private key not known?  rescheduling publish for 30s");

router/java/src/net/i2p/router/networkdb/kademlia/FloodfillNetworkDatabaseFacade.java

@@ -129,6 +129,7 @@ public class FloodfillNetworkDatabaseFacade extends KademliaNetworkDatabaseFacad
      */
     SearchJob search(Hash key, Job onFindJob, Job onFailedLookupJob, long timeoutMs, boolean isLease) {
         //if (true) return super.search(key, onFindJob, onFailedLookupJob, timeoutMs, isLease);
+        if (key == null) throw new IllegalArgumentException("searchin for nothin, eh?");
         boolean isNew = true;
         FloodSearchJob searchJob = null;
         synchronized (_activeFloodQueries) {

router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java

@@ -682,6 +682,12 @@ public class ProfileOrganizer {
                           + "], but there aren't enough of them " + numExceedingMean);
             _thresholdCapacityValue = Math.max(thresholdAtMinHighCap, thresholdAtLowest);
         }
+        
+        // the base growth factor is the value we give to new routers that we don't
+        // know anything about.  dont go under that limit unless you want to expose
+        // the selection to simple ident flooding attacks
+        if (_thresholdCapacityValue <= CapacityCalculator.GROWTH_FACTOR)
+            _thresholdCapacityValue = CapacityCalculator.GROWTH_FACTOR + 0.0001;
     }
     
     /**

router/java/src/net/i2p/router/startup/CreateRouterInfoJob.java

@@ -54,12 +54,8 @@ public class CreateRouterInfoJob extends JobImpl {
             info.setAddresses(getContext().commSystem().createAddresses());
             Properties stats = getContext().statPublisher().publishStatistics();
             stats.setProperty(RouterInfo.PROP_NETWORK_ID, Router.NETWORK_ID+"");
-            getContext().router().addReachabilityCapability(info);
+            getContext().router().addCapabilities(info);
             info.setOptions(stats);
-            if (FloodfillNetworkDatabaseFacade.floodfillEnabled(getContext()))
-                info.addCapability(FloodfillNetworkDatabaseFacade.CAPACITY_FLOODFILL);
-            if ("true".equalsIgnoreCase(getContext().getProperty(Router.PROP_HIDDEN, "false")))
-                info.addCapability(RouterInfo.CAPABILITY_HIDDEN);
             info.setPeers(new HashSet());
             info.setPublished(getCurrentPublishDate(getContext()));
             RouterIdentity ident = new RouterIdentity();

router/java/src/net/i2p/router/startup/RebuildRouterInfoJob.java

@@ -128,14 +128,7 @@ public class RebuildRouterInfoJob extends JobImpl {
                 Properties stats = getContext().statPublisher().publishStatistics();
                 stats.setProperty(RouterInfo.PROP_NETWORK_ID, ""+Router.NETWORK_ID);
                 info.setOptions(stats);
-                if (FloodfillNetworkDatabaseFacade.floodfillEnabled(getContext()))
-                    info.addCapability(FloodfillNetworkDatabaseFacade.CAPACITY_FLOODFILL);
-
-                // Set caps=H for hidden mode routers
-                if ("true".equalsIgnoreCase(getContext().getProperty(Router.PROP_HIDDEN, "false"))) 
-                    info.addCapability(RouterInfo.CAPABILITY_HIDDEN);
-
-                getContext().router().addReachabilityCapability(info);
+                getContext().router().addCapabilities(info);
                 // info.setPeers(new HashSet()); // this would have the trusted peers
                 info.setPublished(CreateRouterInfoJob.getCurrentPublishDate(getContext()));
                 

router/java/src/net/i2p/router/transport/GetBidsJob.java

@@ -47,7 +47,7 @@ public class GetBidsJob extends JobImpl {
         if (context.shitlist().isShitlisted(to)) {
             if (log.shouldLog(Log.WARN))
                 log.warn("Attempt to send a message to a shitlisted peer - " + to);
-            context.messageRegistry().peerFailed(to);
+            //context.messageRegistry().peerFailed(to);
             fail(context, msg);
             return;
         }

router/java/src/net/i2p/router/transport/OutboundMessageRegistry.java

@@ -10,14 +10,8 @@ package net.i2p.router.transport;
 
 import java.io.IOException;
 import java.io.Writer;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
+import java.util.*;
 
-import net.i2p.data.Hash;
 import net.i2p.data.i2np.I2NPMessage;
 import net.i2p.router.Job;
 import net.i2p.router.MessageSelector;
@@ -29,29 +23,25 @@ import net.i2p.util.SimpleTimer;
 
 public class OutboundMessageRegistry {
     private Log _log;
-    /** Expiration date (Long) to OutNetMessage */
-    private TreeMap _pendingMessages;
+    /** list of currently active MessageSelector instances */
+    private List _selectors;
+    /** map of active MessageSelector to either an OutNetMessage or a List of OutNetMessages causing it (for quick removal) */
+    private Map _selectorToMessage;
+    /** set of active OutNetMessage (for quick removal and selector fetching) */
+    private Set _activeMessages;
+    private CleanupTask _cleanupTask;
     private RouterContext _context;
     
-    private final static long CLEANUP_DELAY = 1000*5; // how often to expire pending unreplied messages
-    
     public OutboundMessageRegistry(RouterContext context) {
         _context = context;
         _log = _context.logManager().getLog(OutboundMessageRegistry.class);
-        _pendingMessages = new TreeMap();
-        //_context.jobQueue().addJob(new CleanupPendingMessagesJob());
+        _selectors = new ArrayList(64);
+        _selectorToMessage = new HashMap(64);
+        _activeMessages = new HashSet(64);
+        _cleanupTask = new CleanupTask();
     }
     
-    public void shutdown() {
-        if (_log.shouldLog(Log.WARN)) {
-            StringBuffer buf = new StringBuffer(1024);
-            buf.append("Pending messages: ").append(_pendingMessages.size()).append("\n");
-            for (Iterator iter = _pendingMessages.values().iterator(); iter.hasNext(); ) {
-                buf.append(iter.next().toString()).append("\n\t");
-            }
-            _log.log(Log.WARN, buf.toString());
-        }
-    }
+    public void shutdown() {}
     
     /**
      * Retrieve all messages that are waiting for the specified message.  In
@@ -65,103 +55,72 @@ public class OutboundMessageRegistry {
      *         the payload
      */
     public List getOriginalMessages(I2NPMessage message) {
-        ArrayList matches = new ArrayList(2);
+        ArrayList matchedSelectors = null;
+        ArrayList removedSelectors = null;
         long beforeSync = _context.clock().now();
-
-        Map messages = null;
-        long matchTime = 0;
-        long continueTime = 0;
-        int numMessages = 0;
-        long afterSync1 = 0;
-        long afterSearch = 0;
-        int matchedRemoveCount = 0;
-        StringBuffer slow = null; // new StringBuffer(256);
-
-        synchronized (_pendingMessages) {
-            messages = _pendingMessages; //(Map)_pendingMessages.clone();
-            
-            numMessages = messages.size();
-            afterSync1 = _context.clock().now();
-
-            for (Iterator iter = messages.keySet().iterator(); iter.hasNext(); ) {
-                Long exp = (Long)iter.next();
-                OutNetMessage msg = (OutNetMessage)messages.get(exp);
-                MessageSelector selector = msg.getReplySelector();
-                if (selector != null) {
-                    long before = _context.clock().now();
-                    boolean isMatch = selector.isMatch(message);
-                    long after = _context.clock().now();
-                    long diff = after-before;
-                    if (diff > 100) {
-                        if (_log.shouldLog(Log.WARN))
-                            _log.warn("Matching with selector took too long (" + diff + "ms) : " 
-                                      + selector.getClass().getName());
-                        if (slow == null) slow = new StringBuffer(256);
-                        slow.append(selector.getClass().getName()).append(": ");
-                        slow.append(diff).append(" ");
+        synchronized (_selectors) {
+            for (int i = 0; i < _selectors.size(); i++) {
+                MessageSelector sel = (MessageSelector)_selectors.get(i);
+                if (sel == null)
+                    continue;
+                boolean isMatch = sel.isMatch(message);
+                if (isMatch) {
+                    if (matchedSelectors == null) matchedSelectors = new ArrayList(1);
+                    matchedSelectors.add(sel);
+                    if (!sel.continueMatching()) {
+                        if (removedSelectors == null) removedSelectors = new ArrayList(1);
+                        removedSelectors.add(sel);
+                        _selectors.remove(i);
+                        i--;
                     }
-                    matchTime += diff;
+                }
+            }  
+        }
 
-                    if (isMatch) {
-                        if (_log.shouldLog(Log.DEBUG))
-                            _log.debug("Selector matches [" + selector);
-                        if (!matches.contains(msg))
-                            matches.add(msg);
-                        long beforeCon = _context.clock().now();
-                        boolean continueMatching = selector.continueMatching();
-                        long afterCon = _context.clock().now();
-                        long diffCon = afterCon - beforeCon;
-                        if (diffCon > 100) {
-                            if (_log.shouldLog(Log.WARN))
-                                _log.warn("Error continueMatching on a match took too long (" 
-                                          + diffCon + "ms) : " + selector.getClass().getName());
-                        }
-                        continueTime += diffCon;
-
-                        if (continueMatching) {
-                            if (_log.shouldLog(Log.DEBUG))
-                                _log.debug("Continue matching");
-                            // noop
-                        } else {
-                            if (_log.shouldLog(Log.DEBUG))
-                                _log.debug("Stop matching selector " + selector + " for message " 
-                                           + msg.getMessageType());
-                            
-                            // i give in mihi, i'll use iter.remove just this once ;)
-                            // (TreeMap supports it, and this synchronized block is a hotspot)
-                            iter.remove();
-
-                            matchedRemoveCount++;
-                        }
+        List rv = null;
+        if (matchedSelectors != null) {
+            rv = new ArrayList(matchedSelectors.size());
+            for (int i = 0; i < matchedSelectors.size(); i++) {
+                MessageSelector sel = (MessageSelector)matchedSelectors.get(i);
+                boolean removed = false;
+                OutNetMessage msg = null;
+                List msgs = null;
+                synchronized (_selectorToMessage) {
+                    Object o = null;
+                    if ( (removedSelectors != null) && (removedSelectors.contains(sel)) ) {
+                        o = _selectorToMessage.remove(sel);
+                        removed = true;
                     } else {
-                        //_log.debug("Selector does not match [" + selector + "]");
+                        o = _selectorToMessage.get(sel);
+                    }
+                    
+                    if (o instanceof OutNetMessage) {
+                        msg = (OutNetMessage)o;
+                        if (msg != null)
+                            rv.add(msg);
+                    } else if (o instanceof List) {
+                        msgs = (List)o;
+                        if (msgs != null)
+                            rv.addAll(msgs);
+                    }
+                }
+                if (removed) {
+                    if (msg != null) {
+                        synchronized (_activeMessages) {
+                            _activeMessages.remove(msg);
+                        }
+                    } else if (msgs != null) {
+                        synchronized (_activeMessages) {
+                            _activeMessages.removeAll(msgs);
+                        }
                     }
                 }
             }
-            afterSearch = _context.clock().now();
-        }
-        
-        long delay = _context.clock().now() - beforeSync;
-        long search = afterSearch - afterSync1;
-        long sync = afterSync1 - beforeSync;
-
-        int level = Log.DEBUG;
-        if (delay > 1000)
-            level = Log.ERROR;
-        if (_log.shouldLog(level)) {
-            StringBuffer buf = new StringBuffer(1024);
-            buf.append("getMessages took ").append(delay).append("ms with search time of");
-            buf.append(search).append("ms (match: ").append(matchTime).append("ms, continue: ");
-            buf.append(continueTime).append("ms, #: ").append(numMessages).append(") and sync time of ");
-            buf.append(sync).append("ms for ");
-            buf.append(matchedRemoveCount);
-            buf.append(" removed, ").append(matches.size()).append(" matches: slow = ");
-            if (slow != null)
-                buf.append(slow.toString());
-            _log.log(level, buf.toString());
+        } else {
+            rv = Collections.EMPTY_LIST;
         }
 
-        return matches;
+        return rv;
     }
     
     public OutNetMessage registerPending(MessageSelector replySelector, ReplyJob onReply, Job onTimeout, int timeoutMs) {
@@ -174,271 +133,135 @@ public class OutboundMessageRegistry {
         registerPending(msg, true);
         return msg;
     }
-
     
-    public void registerPending(OutNetMessage msg) {
-        registerPending(msg, false);
-    }
+    public void registerPending(OutNetMessage msg) { registerPending(msg, false); }
     public void registerPending(OutNetMessage msg, boolean allowEmpty) {
-        if (msg == null)
-            throw new IllegalArgumentException("Null OutNetMessage specified?  wtf");
-        if (!allowEmpty) {
-            if (msg.getMessage() == null)
+        if ( (!allowEmpty) && (msg.getMessage() == null) )
                 throw new IllegalArgumentException("OutNetMessage doesn't contain an I2NPMessage? wtf");
+        MessageSelector sel = msg.getReplySelector();
+        if (sel == null) throw new IllegalArgumentException("No reply selector?  wtf");
+
+        boolean alreadyPending = false;
+        synchronized (_activeMessages) {
+            if (!_activeMessages.add(msg))
+                return; // dont add dups
         }
-
-        long beforeSync = _context.clock().now();
-        long afterSync1 = 0;
-        long afterDone = 0;
-        try {
-            OutNetMessage oldMsg = null;
-            long l = msg.getExpiration();
-            synchronized (_pendingMessages) {
-                if (_pendingMessages.containsValue(msg)) {
-                    if (_log.shouldLog(Log.DEBUG))
-                        _log.debug("Not adding an already pending message: " + msg, 
-                                   new Exception("Duplicate message registration"));
-                    return;
-                }               
-
-                while (_pendingMessages.containsKey(new Long(l)))
-                    l++;
-                _pendingMessages.put(new Long(l), msg);
-            }
-            afterSync1 = _context.clock().now();
-            
-            // this may get orphaned if the message is matched or explicitly
-            // removed, but its cheap enough to do an extra remove on the map 
-            // that to poll the list periodically
-            SimpleTimer.getInstance().addEvent(new CleanupExpiredTask(l), l - _context.clock().now());
-
-            if (_log.shouldLog(Log.DEBUG))
-                _log.debug("Register pending: " + msg.getReplySelector().getClass().getName() 
-                           + " for " + msg.getMessage() + ": " 
-                           + msg.getReplySelector().toString(), new Exception("Register pending"));
-            afterDone = _context.clock().now();
-        } finally {
-            long delay = _context.clock().now() - beforeSync;
-            long sync1 = afterSync1 - beforeSync;
-            long done = afterDone - afterSync1;
-            String warn = delay + "ms (sync = " + sync1 + "ms, done = " + done + "ms)";
-            if ( (delay > 1000) && (_log.shouldLog(Log.WARN)) ) {
-                _log.error("Synchronizing in the registry.register took too long!  " + warn);
-                //_context.messageHistory().messageProcessingError(msg.getMessage().getUniqueId(), 
-                //                                                 msg.getMessage().getClass().getName(), 
-                //                                                 "RegisterPending took too long: " + warn);
-            } else {
-                if (_log.shouldLog(Log.DEBUG))
-                    _log.debug("Synchronizing in the registry.register was quick:  " + warn);
+        synchronized (_selectorToMessage) { 
+            Object oldMsg = _selectorToMessage.put(sel, msg);
+            if (oldMsg != null) {
+                List multi = null;
+                if (oldMsg instanceof OutNetMessage) {
+                    //multi = Collections.synchronizedList(new ArrayList(4));
+                    multi = new ArrayList(4);
+                    multi.add(oldMsg);
+                    multi.add(msg);
+                    _selectorToMessage.put(sel, multi);
+                } else if (oldMsg instanceof List) {
+                    multi = (List)oldMsg;
+                    multi.add(msg);
+                    _selectorToMessage.put(sel, multi);
+                }
+                if (_log.shouldLog(Log.WARN))
+                    _log.warn("a single message selector [" + sel + "] with multiple messages ("+ multi + ")");
             }
         }
-        //_log.debug("* Register called of " + msg + "\n\nNow pending are: " + renderStatusHTML(), new Exception("who registered a new one?"));
+        synchronized (_selectors) { _selectors.add(sel); }
+
+        _cleanupTask.scheduleExpiration(sel);
     }
     
     public void unregisterPending(OutNetMessage msg) {
-        long beforeSync = _context.clock().now();
-        try {
-            synchronized (_pendingMessages) {
-                if (_pendingMessages.containsValue(msg)) {
-                    Long found = null;
-                    for (Iterator iter = _pendingMessages.keySet().iterator(); iter.hasNext();) {
-                        Long exp = (Long)iter.next();
-                        Object val = _pendingMessages.get(exp);
-                        if (val.equals(msg)) {
-                            found = exp;
-                            break;
-                        }
-                    }
-                    
-                    if (found != null) {
-                        if (_log.shouldLog(Log.DEBUG))
-                            _log.debug("Unregistered message " + msg.getReplySelector() 
-                                       + ": " + msg, new Exception("Who unregistered?"));
-                        _pendingMessages.remove(found);
-                    } else {
-                        _log.error("Arg, couldn't find the message that we... thought we could find?", 
-                                   new Exception("WTF"));
+        MessageSelector sel = msg.getReplySelector();
+        boolean stillActive = false;
+        synchronized (_selectorToMessage) { 
+            Object old = _selectorToMessage.remove(sel);
+            if (old != null) {
+                if (old instanceof List) {
+                    List l = (List)old;
+                    l.remove(msg);
+                    if (l.size() > 0) {
+                        _selectorToMessage.put(sel, l);
+                        stillActive = true;
                     }
                 }
             }
-        } finally {
-            long delay = _context.clock().now() - beforeSync;
-            String warn = delay + "ms";
-            if ( (delay > 1000) && (_log.shouldLog(Log.WARN)) ) {
-                _log.warn("Synchronizing in the registry.unRegister took too long!  " + warn);
-                _context.messageHistory().messageProcessingError(msg.getMessageId(), msg.getMessageType(), "Unregister took too long: " + warn);
-            } else {
-                if (_log.shouldLog(Log.DEBUG)) 
-                    _log.debug("Synchronizing in the registry.unRegister was quick:  " + warn);
-            }
         }
+        if (!stillActive)
+            synchronized (_selectors) { _selectors.remove(sel); }
+        synchronized (_activeMessages) { _activeMessages.remove(msg); }
     }
 
-    public void peerFailed(Hash peer) {
-        List failed = null;
-        int numFailed = 0;
-        synchronized (_pendingMessages) {
-            for (Iterator iter = _pendingMessages.values().iterator(); iter.hasNext(); ) {
-                OutNetMessage msg = (OutNetMessage)iter.next();
-                if (msg.getTarget() != null) {
-                    Hash to = msg.getTarget().getIdentity().calculateHash();
-                    if (to.equals(peer)) {
-                        if (failed == null)
-                            failed = new ArrayList(4);
-                        failed.add(msg);
-                        iter.remove();
-                        numFailed++;
-                    } else {
-                        if (_log.shouldLog(Log.DEBUG))
-                            _log.debug("Peer failed: " + peer.toBase64().substring(0,6) 
-                                       + " but not killing a message to " 
-                                       + to.toBase64().substring(0,6));
-                    }
-                }
-            }
-        }
-        if (failed != null) {
-            for (int i = 0; i < failed.size(); i++) {
-                OutNetMessage msg = (OutNetMessage)failed.get(i);
-                msg.discardData();
-                if (msg.getOnFailedSendJob() != null)
-                    _context.jobQueue().addJob(msg.getOnFailedSendJob());
-            }
-        }
-        
-        if (_log.shouldLog(Log.WARN))
-            _log.warn("Peer failed: " + peer.toBase64().substring(0,6) + " killing " + numFailed);
-    }
+    public void renderStatusHTML(Writer out) throws IOException {}
     
-    public void renderStatusHTML(Writer out) throws IOException {
-        StringBuffer buf = new StringBuffer(8192);
-        buf.append("<h2>Pending messages</h2>\n");
-        Map msgs = null;
-        synchronized (_pendingMessages) {
-            msgs = (Map)_pendingMessages.clone();
-        }
-        buf.append("<ul>");
-        for (Iterator iter = msgs.keySet().iterator(); iter.hasNext();) {
-            Long exp = (Long)iter.next();
-            OutNetMessage msg = (OutNetMessage)msgs.get(exp);
-            buf.append("<li>").append(msg.getMessageType());
-            buf.append(": expiring on ").append(new Date(exp.longValue()));
-            if (msg.getTarget() != null)
-                buf.append(" targetting ").append(msg.getTarget().getIdentity().getHash());
-            if (msg.getReplySelector() != null)
-                buf.append(" with reply selector ").append(msg.getReplySelector().toString());
-            else
-                buf.append(" with NO reply selector?  WTF!");
-            buf.append("</li>\n");
-        }
-        buf.append("</ul>");
-        out.write(buf.toString());
-        out.flush();
-    }
-    
-    private class CleanupExpiredTask implements SimpleTimer.TimedEvent {
-        private long _expiration;
-        public CleanupExpiredTask(long expiration) {
-            _expiration = expiration;
+    private class CleanupTask implements SimpleTimer.TimedEvent {
+        private List _removing;
+        private long _nextExpire;
+        public CleanupTask() {
+            _removing = new ArrayList(4);
+            _nextExpire = -1;
         }
         public void timeReached() {
-            OutNetMessage msg = null;
-            synchronized (_pendingMessages) {
-                msg = (OutNetMessage)_pendingMessages.remove(new Long(_expiration));
-            }
-            if (msg != null) {
-                _context.messageHistory().replyTimedOut(msg);
-                Job fail = msg.getOnFailedReplyJob();
-                if (fail != null) {
-                    if (_log.shouldLog(Log.DEBUG))
-                        _log.debug("Removing message with selector " + msg.getReplySelector() 
-                                   + ": " + msg.getMessageType() 
-                                   + " and firing fail job: " + fail.getClass().getName());
-                    _context.jobQueue().addJob(fail);
-                } else {
-                    if (_log.shouldLog(Log.DEBUG))
-                        _log.debug("Removing message with selector " + msg.getReplySelector() 
-                                   + " and not firing any job");
+            long now = _context.clock().now();
+            synchronized (_selectors) {
+                for (int i = 0; i < _selectors.size(); i++) {
+                    MessageSelector sel = (MessageSelector)_selectors.get(i);
+                    if (sel == null) continue;
+                    long expiration = sel.getExpiration();
+                    if (expiration <= now) {
+                        _removing.add(sel);
+                        _selectors.remove(i);
+                        i--;
+                    } else if (expiration < _nextExpire || _nextExpire < now) {
+                        _nextExpire = expiration;
+                    }
                 }
             }
+            if (_removing.size() > 0) {
+                for (int i = 0; i < _removing.size(); i++) {
+                    MessageSelector sel = (MessageSelector)_removing.get(i);
+                    OutNetMessage msg = null;
+                    List msgs = null;
+                    synchronized (_selectorToMessage) {
+                        Object o = _selectorToMessage.remove(sel);
+                        if (o instanceof OutNetMessage) {
+                            msg = (OutNetMessage)o;
+                        } else if (o instanceof List) {
+                            //msgs = new ArrayList((List)o);
+                            msgs = (List)o;
+                        }
+                    }
+                    if (msg != null) {
+                        synchronized (_activeMessages) {
+                            _activeMessages.remove(msg);
+                        }
+                        Job fail = msg.getOnFailedReplyJob();
+                        if (fail != null)
+                            _context.jobQueue().addJob(fail);
+                    } else if (msgs != null) {
+                        synchronized (_activeMessages) {
+                            _activeMessages.removeAll(msgs);
+                        }
+                        for (int j = 0; j < msgs.size(); j++) {
+                            msg = (OutNetMessage)msgs.get(j);
+                            Job fail = msg.getOnFailedReplyJob();
+                            if (fail != null)
+                                _context.jobQueue().addJob(fail);
+                        }
+                    }
+                }
+                _removing.clear();
+            }
+
+            if (_nextExpire <= now)
+                _nextExpire = now + 10*1000;
+            SimpleTimer.getInstance().addEvent(CleanupTask.this, _nextExpire - now);
+        }
+        public void scheduleExpiration(MessageSelector sel) {
+            long now = _context.clock().now();
+            if ( (_nextExpire <= now) || (sel.getExpiration() < _nextExpire) ) {
+                _nextExpire = sel.getExpiration();
+                SimpleTimer.getInstance().addEvent(CleanupTask.this, _nextExpire - now);
+            }
         }
     }
-
-    /**
-     * Cleanup any messages that were pending replies but have expired
-     *
-     */
-    /*
-    private class CleanupPendingMessagesJob extends JobImpl { 
-        public CleanupPendingMessagesJob() {
-            super(OutboundMessageRegistry.this._context);
-        }
-
-        public String getName() { return "Cleanup any messages that timed out"; }
-        
-        public void runJob() {
-            List removed = removeMessages();
-            
-            RouterContext ctx = OutboundMessageRegistry.this._context;
-                
-            for (int i = 0; i < removed.size(); i++) {
-                OutNetMessage msg = (OutNetMessage)removed.get(i);
-
-                if (msg != null) {
-                    _context.messageHistory().replyTimedOut(msg);
-                    Job fail = msg.getOnFailedReplyJob();
-                    if (fail != null) {
-                        if (_log.shouldLog(Log.DEBUG))
-                            _log.debug("Removing message with selector " + msg.getReplySelector() 
-                                       + ": " + msg.getMessageType() 
-                                       + " and firing fail job: " + fail.getClass().getName());
-                        _context.jobQueue().addJob(fail);
-                    } else {
-                        if (_log.shouldLog(Log.DEBUG))
-                            _log.debug("Removing message with selector " + msg.getReplySelector() 
-                                       + " and not firing any job");
-                    }
-                }
-            }
-
-            requeue(CLEANUP_DELAY);
-        }
-        
-        /**
-         * Remove any messages whose expirations are in the past
-         *
-         * @return list of OutNetMessage objects that have expired
-         */ /*
-        private List removeMessages() {
-            long now = OutboundMessageRegistry.this._context.clock().now();
-            List removedMessages = new ArrayList(2);
-            List expirationsToRemove = null;
-            synchronized (_pendingMessages) {
-                for (Iterator iter = _pendingMessages.keySet().iterator(); iter.hasNext();) {
-                    Long expiration = (Long)iter.next();
-                    if (expiration.longValue() < now) {
-                        if (expirationsToRemove == null)
-                            expirationsToRemove = new ArrayList(8);
-                        expirationsToRemove.add(expiration);
-                    } else {
-                        // its sorted
-                        break;
-                    }
-                }
-
-                if (expirationsToRemove != null) {
-                    for (int i = 0; i < expirationsToRemove.size(); i++) {
-                        Long expiration = (Long)expirationsToRemove.get(i);
-                        OutNetMessage msg = (OutNetMessage)_pendingMessages.remove(expiration);
-                        if (msg != null)
-                            removedMessages.add(msg);
-                    }
-                }
-            }
-            if (_log.shouldLog(Log.DEBUG))
-                _log.debug("Removed " + removedMessages.size() + " messages");
-            return removedMessages;
-        }
-    }
-    */
 }

router/java/src/net/i2p/router/transport/TransportImpl.java

@@ -203,6 +203,8 @@ public abstract class TransportImpl implements Transport {
                               + msg.getMessageType() + " message with selector " + selector, new Exception("fail cause"));
                 if (msg.getOnFailedSendJob() != null)
                     _context.jobQueue().addJob(msg.getOnFailedSendJob());
+                if (msg.getOnFailedReplyJob() != null)
+                    _context.jobQueue().addJob(msg.getOnFailedReplyJob());
                 if (selector != null)
                     _context.messageRegistry().unregisterPending(msg);
                 log = true;

router/java/src/net/i2p/router/transport/udp/EstablishmentManager.java

@@ -552,7 +552,11 @@ public class EstablishmentManager {
             // offer to relay
             // (perhaps we should check our bw usage and/or how many peers we are 
             //  already offering introducing?)
-            state.setSentRelayTag(_context.random().nextLong(MAX_TAG_VALUE));
+            if (state.getSentRelayTag() < 0) {
+                state.setSentRelayTag(_context.random().nextLong(MAX_TAG_VALUE));
+            } else {
+                // don't change it, since we've already prepared our sig
+            }
         } else {
             // don't offer to relay
             state.setSentRelayTag(0);

router/java/src/net/i2p/router/transport/udp/MessageReceiver.java

@@ -156,8 +156,11 @@ public class MessageReceiver {
                                + " raw: " + Base64.encode(fragments[i].getData()) + ")");
                 off += fragments[i].getValid();
             }
-            if (off != state.getCompleteSize())
-                _log.error("Hmm, offset of the fragments = " + off + " while the state says " + state.getCompleteSize());
+            if (off != state.getCompleteSize()) {
+                if (_log.shouldLog(Log.WARN))
+                    _log.warn("Hmm, offset of the fragments = " + off + " while the state says " + state.getCompleteSize());
+                return null;
+            }
             if (_log.shouldLog(Log.DEBUG))
                 _log.debug("Raw byte array for " + state.getMessageId() + ": " + Base64.encode(buf.getData(), 0, state.getCompleteSize()));
             I2NPMessage m = I2NPMessageImpl.fromRawByteArray(_context, buf.getData(), 0, state.getCompleteSize(), handler);

router/java/src/net/i2p/router/transport/udp/OutboundMessageFragments.java

@@ -61,7 +61,7 @@ public class OutboundMessageFragments {
         _context.statManager().createRateStat("udp.sendConfirmVolley", "How many times did fragments need to be sent before ACK", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
         _context.statManager().createRateStat("udp.sendFailed", "How many sends a failed message was pushed", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
         _context.statManager().createRateStat("udp.sendAggressiveFailed", "How many volleys was a packet sent before we gave up", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
-        _context.statManager().createRateStat("udp.outboundActiveCount", "How many messages are in the active pool", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
+        _context.statManager().createRateStat("udp.outboundActiveCount", "How many messages are in the peer's active pool", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
         _context.statManager().createRateStat("udp.sendRejected", "What volley are we on when the peer was throttled (time == message lifetime)", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
         _context.statManager().createRateStat("udp.partialACKReceived", "How many fragments were partially ACKed (time == message lifetime)", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
         _context.statManager().createRateStat("udp.sendSparse", "How many fragments were partially ACKed and hence not resent (time == message lifetime)", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });

router/java/src/net/i2p/router/transport/udp/OutboundMessageState.java

@@ -47,8 +47,7 @@ public class OutboundMessageState {
     public boolean initialize(OutNetMessage msg) {
         if (msg == null) return false;
         try {
-            initialize(msg, msg.getMessage(), null);
-            return true;
+            return initialize(msg, msg.getMessage(), null);
         } catch (OutOfMemoryError oom) {
             throw oom;
         } catch (Exception e) {
@@ -62,8 +61,7 @@ public class OutboundMessageState {
             return false;
         
         try {
-            initialize(null, msg, peer);
-            return true;
+            return initialize(null, msg, peer);
         } catch (OutOfMemoryError oom) {
             throw oom;
         } catch (Exception e) {
@@ -77,8 +75,7 @@ public class OutboundMessageState {
             return false;
         
         try {
-            initialize(m, msg, null);
-            return true;
+            return initialize(m, msg, null);
         } catch (OutOfMemoryError oom) {
             throw oom;
         } catch (Exception e) {
@@ -87,7 +84,7 @@ public class OutboundMessageState {
         }
     }
     
-    private void initialize(OutNetMessage m, I2NPMessage msg, PeerState peer) {
+    private boolean initialize(OutNetMessage m, I2NPMessage msg, PeerState peer) {
         _message = m;
         _peer = peer;
         if (_messageBuf != null) {
@@ -99,17 +96,24 @@ public class OutboundMessageState {
         int size = msg.getRawMessageSize();
         if (size > _messageBuf.getData().length)
             throw new IllegalArgumentException("Size too large!  " + size + " in " + msg);
-        int len = msg.toRawByteArray(_messageBuf.getData());
-        _messageBuf.setValid(len);
-        _messageId = msg.getUniqueId();
-        
-        _startedOn = _context.clock().now();
-        _nextSendTime = _startedOn;
-        _expiration = _startedOn + 10*1000;
-        //_expiration = msg.getExpiration();
-        
-        if (_log.shouldLog(Log.DEBUG))
-            _log.debug("Raw byte array for " + _messageId + ": " + Base64.encode(_messageBuf.getData(), 0, len));
+        try {
+            int len = msg.toRawByteArray(_messageBuf.getData());
+            _messageBuf.setValid(len);
+            _messageId = msg.getUniqueId();
+
+            _startedOn = _context.clock().now();
+            _nextSendTime = _startedOn;
+            _expiration = _startedOn + 10*1000;
+            //_expiration = msg.getExpiration();
+
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("Raw byte array for " + _messageId + ": " + Base64.encode(_messageBuf.getData(), 0, len));
+            return true;
+        } catch (IllegalStateException ise) {
+            _cache.release(_messageBuf);
+            _messageBuf = null;
+            return false;
+        }
     }
     
     public void releaseResources() { 

router/java/src/net/i2p/router/transport/udp/PacketBuilder.java

@@ -567,7 +567,7 @@ public class PacketBuilder {
             if ( (off % 16) != 0)
                 off += 16 - (off % 16);
             packet.getPacket().setLength(off);
-            authenticate(packet, state.getIntroKey(), state.getIntroKey());
+            authenticate(packet, state.getCipherKey(), state.getMACKey());
         } 
         
         setTo(packet, to, state.getSentPort());

router/java/src/net/i2p/router/transport/udp/PeerState.java

@@ -10,19 +10,19 @@ import java.util.Map;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 
-import net.i2p.I2PAppContext;
 import net.i2p.data.Hash;
 import net.i2p.data.SessionKey;
 import net.i2p.util.Log;
 import net.i2p.router.RouterContext;
 import net.i2p.router.OutNetMessage;
+import net.i2p.router.Job;
 
 /**
  * Contain all of the state about a UDP connection to a peer.
  *
  */
 public class PeerState {
-    private I2PAppContext _context;
+    private RouterContext _context;
     private Log _log;
     /**
      * The peer are we talking to.  This should be set as soon as this
@@ -216,7 +216,7 @@ public class PeerState {
     /** override the default MTU */
     private static final String PROP_DEFAULT_MTU = "i2np.udp.mtu";
     
-    public PeerState(I2PAppContext ctx, UDPTransport transport) {
+    public PeerState(RouterContext ctx, UDPTransport transport) {
         _context = ctx;
         _log = ctx.logManager().getLog(PeerState.class);
         _transport = transport;
@@ -278,6 +278,8 @@ public class PeerState {
         _context.statManager().createRateStat("udp.rejectConcurrentActive", "How many messages are currently being sent to the peer when we reject it (period is how many concurrent packets we allow)", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
         _context.statManager().createRateStat("udp.allowConcurrentActive", "How many messages are currently being sent to the peer when we accept it (period is how many concurrent packets we allow)", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
         _context.statManager().createRateStat("udp.rejectConcurrentSequence", "How many consecutive concurrency rejections have we had when we stop rejecting (period is how many concurrent packets we are on)", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
+        _context.statManager().createRateStat("udp.queueDropSize", "How many messages were queued up when it was considered full, causing a tail drop?", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
+        _context.statManager().createRateStat("udp.queueAllowTotalLifetime", "When a peer is retransmitting and we probabalistically allow a new message, what is the sum of the pending message lifetimes? (period is the new message's lifetime)?", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
     }
     
     private int getDefaultMTU() {
@@ -874,7 +876,7 @@ public class PeerState {
         double retransPct = 0;
         if (_packetsTransmitted > 10) {
             retransPct = (double)_packetsRetransmitted/(double)_packetsTransmitted;
-            boolean wantLarge = retransPct < .50d; // heuristic to allow fairly lossy links to use large MTUs
+            boolean wantLarge = retransPct < .30d; // heuristic to allow fairly lossy links to use large MTUs
             if (wantLarge && _mtu != LARGE_MTU) {
                 if (_context.random().nextLong(_mtuDecreases) <= 0) {
                     _mtu = LARGE_MTU;
@@ -997,7 +999,6 @@ public class PeerState {
     }
     
     public RemoteHostId getRemoteHostId() { return _remoteHostId; }
-
     
     public int add(OutboundMessageState state) {
         if (_dead) { 
@@ -1009,10 +1010,54 @@ public class PeerState {
             _log.debug("Adding to " + _remotePeer.toBase64() + ": " + state.getMessageId());
         List msgs = _outboundMessages;
         if (msgs == null) return 0;
+        int rv = 0;
+        boolean fail = false;
         synchronized (msgs) {
-            msgs.add(state);
-            return msgs.size();
+            rv = msgs.size() + 1;
+            if (rv > 32) { 
+                // 32 queued messages?  to *one* peer?  nuh uh.
+                fail = true;
+                rv--;
+            } else if (_retransmitter != null) {
+                long lifetime = _retransmitter.getLifetime();
+                long totalLifetime = lifetime;
+                for (int i = 1; i < msgs.size(); i++) { // skip the first, as thats the retransmitter
+                    OutboundMessageState cur = (OutboundMessageState)msgs.get(i);
+                    totalLifetime += cur.getLifetime();
+                }
+                long remaining = -1;
+                OutNetMessage omsg = state.getMessage();
+                if (omsg != null)
+                    remaining = omsg.getExpiration() - _context.clock().now();
+                else
+                    remaining = 10*1000 - state.getLifetime();
+                
+                if (remaining <= 0)
+                    remaining = 1; // total lifetime will exceed it anyway, guaranteeing failure
+                float pDrop = totalLifetime / (float)remaining;
+                pDrop = pDrop * pDrop * pDrop;
+                if (pDrop >= _context.random().nextFloat()) {
+                    if (_log.shouldLog(Log.WARN))
+                        _log.warn("Proactively tail dropping for " + _remotePeer.toBase64() + " (messages=" + msgs.size() 
+                                  + " headLifetime=" + lifetime + " totalLifetime=" + totalLifetime + " curLifetime=" + state.getLifetime() 
+                                  + " remaining=" + remaining + " pDrop=" + pDrop + ")");
+                    _context.statManager().addRateData("udp.queueDropSize", msgs.size(), totalLifetime);
+                    fail = true;
+                } else { 
+                    if (_log.shouldLog(Log.DEBUG))
+                        _log.debug("Probabalistically allowing for " + _remotePeer.toBase64() + " (messages=" + msgs.size() 
+                                   + " headLifetime=" + lifetime + " totalLifetime=" + totalLifetime + " curLifetime=" + state.getLifetime() 
+                                   + " remaining=" + remaining + " pDrop=" + pDrop + ")");
+                    _context.statManager().addRateData("udp.queueAllowTotalLifetime", totalLifetime, lifetime);
+                    msgs.add(state);
+                }
+            } else {
+                msgs.add(state);
+            }
         }
+        if (fail)
+            _transport.failed(state, false);
+        return rv;
     }
     /** drop all outbound messages */
     public void dropOutbound() {
@@ -1202,7 +1247,7 @@ public class PeerState {
      * mind bw/cwin throttle, etc)
      *
      */
-    private static final boolean THROTTLE_INITIAL_SEND = false;
+    private static final boolean THROTTLE_INITIAL_SEND = true;
     
     private static final int SSU_HEADER_SIZE = 46;
     static final int UDP_HEADER_SIZE = 8;

router/java/src/net/i2p/router/transport/udp/UDPTransport.java

@@ -337,6 +337,7 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
                 if ( (_externalListenHost == null) ||
                      (!eq(_externalListenHost.getAddress(), _externalListenPort, ourIP, ourPort)) ) {
                     if ( (_reachabilityStatus == CommSystemFacade.STATUS_UNKNOWN) ||
+                         (_externalListenHost == null) || (_externalListenPort <= 0) ||
                          (_context.clock().now() - _reachabilityStatusLastUpdated > 2*TEST_FREQUENCY) ) {
                         // they told us something different and our tests are either old or failing
                         if (_log.shouldLog(Log.INFO))
@@ -358,7 +359,7 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
                         // so lets test again
                         fireTest = true;
                         if (_log.shouldLog(Log.INFO))
-                            _log.info("Different address, but we're fine..");
+                            _log.info("Different address, but we're fine.. (" + _reachabilityStatus + ")");
                     }
                 } else {
                     // matched what we expect

router/java/src/net/i2p/router/tunnel/TunnelDispatcher.java

@@ -435,7 +435,7 @@ public class TunnelDispatcher implements Service {
             _context.statManager().addRateData("tunnel.dispatchInbound", 1, 0);
         } else {
             _context.messageHistory().droppedTunnelGatewayMessageUnknown(msg.getUniqueId(), msg.getTunnelId().getTunnelId());
-            int level = (_context.router().getUptime() > 10*60*1000 ? Log.ERROR : Log.WARN);
+            int level = (_context.router().getUptime() > 10*60*1000 ? Log.WARN : Log.INFO);
             if (_log.shouldLog(level))
                 _log.log(level, "no matching tunnel for id=" + msg.getTunnelId().getTunnelId() 
                            + ": gateway message expiring in " 

router/java/src/net/i2p/router/tunnel/pool/BuildExecutor.java

@@ -41,6 +41,7 @@ class BuildExecutor implements Runnable {
         _context.statManager().createRateStat("tunnel.buildClientReject", "How often a client tunnel is rejected", "Tunnels", new long[] { 60*1000, 10*60*1000 });
         _context.statManager().createRateStat("tunnel.buildRequestTime", "How long it takes to build a tunnel request", "Tunnels", new long[] { 60*1000, 10*60*1000 });
         _context.statManager().createRateStat("tunnel.buildRequestZeroHopTime", "How long it takes to build a zero hop tunnel", "Tunnels", new long[] { 60*1000, 10*60*1000 });
+        _context.statManager().createRateStat("tunnel.pendingRemaining", "How many inbound requests are pending after a pass (period is how long the pass takes)?", "Tunnels", new long[] { 60*1000, 10*60*1000 });
         _repoll = false;
         _handler = new BuildHandler(ctx, this);
     }
@@ -117,11 +118,19 @@ class BuildExecutor implements Runnable {
         List wanted = new ArrayList(8);
         List pools = new ArrayList(8);
         
-        boolean pendingRemaining = false;
+        int pendingRemaining = 0;
         
+        long loopBegin = 0;
+        long beforeHandleInboundReplies = 0;
+        long afterHandleInboundReplies = 0;
+        long afterBuildZeroHop = 0;
+        long afterBuildReal = 0;
+        long afterHandleInbound = 0;
+                
         while (!_manager.isShutdown()){
+            loopBegin = System.currentTimeMillis();
             try {
-                _repoll = pendingRemaining; // resets repoll to false unless there are inbound requeusts pending
+                _repoll = pendingRemaining > 0; // resets repoll to false unless there are inbound requeusts pending
                 _manager.listPools(pools);
                 for (int i = 0; i < pools.size(); i++) {
                     TunnelPool pool = (TunnelPool)pools.get(i);
@@ -130,7 +139,9 @@ class BuildExecutor implements Runnable {
                         wanted.add(pool);
                 }
 
+                beforeHandleInboundReplies = System.currentTimeMillis();
                 _handler.handleInboundReplies();
+                afterHandleInboundReplies = System.currentTimeMillis();
                 
                 // allowed() also expires timed out requests (for new style requests)
                 int allowed = allowed();
@@ -140,17 +151,22 @@ class BuildExecutor implements Runnable {
 
                 // zero hop ones can run inline
                 allowed = buildZeroHopTunnels(wanted, allowed);
+                afterBuildZeroHop = System.currentTimeMillis();
                 
                 if (_log.shouldLog(Log.DEBUG))
                     _log.debug("Zero hops built, Allowed: " + allowed + " wanted: " + wanted);
 
+                int realBuilt = 0;
                 TunnelManagerFacade mgr = _context.tunnelManager();
                 if ( (mgr == null) || (mgr.selectInboundTunnel() == null) || (mgr.selectOutboundTunnel() == null) ) {
                     // we don't have either inbound or outbound tunnels, so don't bother trying to build
                     // non-zero-hop tunnels
                     synchronized (_currentlyBuilding) {
-                        if (!_repoll)
-                            _currentlyBuilding.wait(5*1000+_context.random().nextInt(5*1000));
+                        if (!_repoll) {
+                            if (_log.shouldLog(Log.DEBUG))
+                                _log.debug("No tunnel to build with (allowed=" + allowed + ", wanted=" + wanted.size() + ", pending=" + pendingRemaining + "), wait for a while");
+                            _currentlyBuilding.wait(1*1000+_context.random().nextInt(1*1000));
+                        }
                     }
                 } else {
                     if ( (allowed > 0) && (wanted.size() > 0) ) {
@@ -173,6 +189,7 @@ class BuildExecutor implements Runnable {
                                     _currentlyBuilding.add(cfg);
                                 }
                                 buildTunnel(pool, cfg);
+                                realBuilt++;
                                 // 0hops are taken care of above, these are nonstandard 0hops
                                 //if (cfg.getLength() <= 1)
                                 //    i--; //0hop, we can keep going, as there's no worry about throttling
@@ -184,13 +201,13 @@ class BuildExecutor implements Runnable {
                             }
                         }
                     } else {
-                        if (_log.shouldLog(Log.DEBUG))
-                            _log.debug("Nothin' doin, wait for a while");
                         try {
                             synchronized (_currentlyBuilding) {
                                 if (!_repoll) {
+                                    if (_log.shouldLog(Log.DEBUG))
+                                        _log.debug("Nothin' doin (allowed=" + allowed + ", wanted=" + wanted.size() + ", pending=" + pendingRemaining + "), wait for a while");
                                     //if (allowed <= 0)
-                                        _currentlyBuilding.wait(_context.random().nextInt(5*1000));
+                                        _currentlyBuilding.wait(_context.random().nextInt(1*1000));
                                     //else // wanted <= 0
                                     //    _currentlyBuilding.wait(_context.random().nextInt(30*1000));
                                 }
@@ -201,8 +218,23 @@ class BuildExecutor implements Runnable {
                     }
                 }
                 
+                afterBuildReal = System.currentTimeMillis();
+                
                 pendingRemaining = _handler.handleInboundRequests();
+                afterHandleInbound = System.currentTimeMillis();
+                
+                if (pendingRemaining > 0)
+                    _context.statManager().addRateData("tunnel.pendingRemaining", pendingRemaining, afterHandleInbound-afterBuildReal);
 
+                if (_log.shouldLog(Log.INFO))
+                    _log.info("build loop complete, tot=" + (afterHandleInbound-loopBegin) + 
+                              " inReply=" + (afterHandleInboundReplies-beforeHandleInboundReplies) +
+                              " zeroHop=" + (afterBuildZeroHop-afterHandleInboundReplies) +
+                              " real=" + (afterBuildReal-afterBuildZeroHop) +
+                              " in=" + (afterHandleInbound-afterBuildReal) + 
+                              " built=" + realBuilt +
+                              " pending=" + pendingRemaining);
+                
                 wanted.clear();
                 pools.clear();
             } catch (Exception e) {
@@ -300,4 +332,5 @@ class BuildExecutor implements Runnable {
     }
     
     List locked_getCurrentlyBuilding() { return _currentlyBuilding; }
+    public int getInboundBuildQueueSize() { return _handler.getInboundBuildQueueSize(); }
 }

router/java/src/net/i2p/router/tunnel/pool/BuildHandler.java

@@ -6,6 +6,8 @@ import net.i2p.data.i2np.*;
 import net.i2p.router.*;
 import net.i2p.router.tunnel.*;
 import net.i2p.router.peermanager.TunnelHistory;
+import net.i2p.stat.Rate;
+import net.i2p.stat.RateStat;
 import net.i2p.util.Log;
 
 /**
@@ -44,6 +46,9 @@ class BuildHandler {
         _context.statManager().createRateStat("tunnel.rejectOverloaded", "How long we had to wait before processing the request (when it was rejected)", "Tunnels", new long[] { 60*1000, 10*60*1000 });
         _context.statManager().createRateStat("tunnel.acceptLoad", "Delay before processing the accepted request", "Tunnels", new long[] { 60*1000, 10*60*1000 });
         _context.statManager().createRateStat("tunnel.dropLoad", "How long we had to wait before finally giving up on an inbound request (period is queue count)?", "Tunnels", new long[] { 60*1000, 10*60*1000 });
+        _context.statManager().createRateStat("tunnel.dropLoadDelay", "How long we had to wait before finally giving up on an inbound request?", "Tunnels", new long[] { 60*1000, 10*60*1000 });
+        _context.statManager().createRateStat("tunnel.dropLoadBacklog", "How many requests were pending when they were so lagged that we had to drop a new inbound request??", "Tunnels", new long[] { 60*1000, 10*60*1000 });
+        _context.statManager().createRateStat("tunnel.dropLoadProactive", "What the estimated queue time was when we dropped an inbound request (period is num pending)", "Tunnels", new long[] { 60*1000, 10*60*1000 });
         _context.statManager().createRateStat("tunnel.handleRemaining", "How many pending inbound requests were left on the queue after one pass?", "Tunnels", new long[] { 60*1000, 10*60*1000 });
         
         _context.statManager().createRateStat("tunnel.receiveRejectionProbabalistic", "How often we are rejected probabalistically?", "Tunnels", new long[] { 10*60*1000l, 60*60*1000l, 24*60*60*1000l });
@@ -62,10 +67,11 @@ class BuildHandler {
     private static final int NEXT_HOP_LOOKUP_TIMEOUT = 5*1000;
     
     /**
-     * Blocking call to handle a few of the pending inbound requests, returning true if
-     * there are remaining requeusts we skipped over
+     * Blocking call to handle a few of the pending inbound requests, returning how many
+     * requests remain after this pass
      */
-    boolean handleInboundRequests() {
+    int handleInboundRequests() {
+        int dropExpired = 0;
         List handled = null;
         synchronized (_inboundBuildMessages) {
             int toHandle = _inboundBuildMessages.size();
@@ -73,8 +79,31 @@ class BuildHandler {
                 if (toHandle > MAX_HANDLE_AT_ONCE)
                     toHandle = MAX_HANDLE_AT_ONCE;
                 handled = new ArrayList(toHandle);
-                for (int i = 0; i < toHandle; i++) // LIFO for lower response time (should we RED it for DoS?)
-                    handled.add(_inboundBuildMessages.remove(_inboundBuildMessages.size()-1));
+                if (false) {
+                    for (int i = 0; i < toHandle; i++) // LIFO for lower response time (should we RED it for DoS?)
+                        handled.add(_inboundBuildMessages.remove(_inboundBuildMessages.size()-1));
+                } else {
+                    // drop any expired messages
+                    long dropBefore = System.currentTimeMillis() - BuildRequestor.REQUEST_TIMEOUT;
+                    do {
+                        BuildMessageState state = (BuildMessageState)_inboundBuildMessages.get(0);
+                        if (state.recvTime <= dropBefore) {
+                            _inboundBuildMessages.remove(0);
+                            dropExpired++;
+                            if (_log.shouldLog(Log.WARN))
+                                _log.warn("Not even trying to handle/decrypt the request " + state.msg.getUniqueId() 
+                                           + ", since we received it a long time ago: " + (System.currentTimeMillis() - state.recvTime));
+                            _context.statManager().addRateData("tunnel.dropLoadDelay", System.currentTimeMillis() - state.recvTime, 0);
+                        } else {
+                            break;
+                        }
+                    } while (_inboundBuildMessages.size() > 0);
+                    
+                    // now pull off the oldest requests first (we're doing a tail-drop
+                    // when adding)
+                    for (int i = 0; i < toHandle && _inboundBuildMessages.size() > 0; i++)
+                        handled.add(_inboundBuildMessages.remove(0));
+                }
             }
         }
         if (handled != null) {
@@ -113,7 +142,7 @@ class BuildHandler {
             int remaining = _inboundBuildMessages.size();
             if (remaining > 0)
                 _context.statManager().addRateData("tunnel.handleRemaining", remaining, 0);
-            return remaining > 0;
+            return remaining;
         }
     }
     
@@ -246,11 +275,12 @@ class BuildHandler {
         if (_log.shouldLog(Log.DEBUG))
             _log.debug(state.msg.getUniqueId() + ": handling request after " + timeSinceReceived);
         
-        if (timeSinceReceived > BuildRequestor.REQUEST_TIMEOUT*2) {
+        if (timeSinceReceived > BuildRequestor.REQUEST_TIMEOUT) {
             // don't even bother, since we are so overloaded locally
-            if (_log.shouldLog(Log.ERROR))
-                _log.error("Not even trying to handle/decrypt the request " + state.msg.getUniqueId() 
+            if (_log.shouldLog(Log.WARN))
+                _log.warn("Not even trying to handle/decrypt the request " + state.msg.getUniqueId() 
                            + ", since we received it a long time ago: " + timeSinceReceived);
+            _context.statManager().addRateData("tunnel.dropLoadDelay", timeSinceReceived, 0);
             return;
         }
         // ok, this is not our own tunnel, so we need to do some heavy lifting
@@ -337,6 +367,30 @@ class BuildHandler {
         }
     }
     
+    /**
+     * If we are dropping lots of requests before even trying to handle them,
+     * I suppose you could call us "overloaded"
+     */
+    private final static int MAX_PROACTIVE_DROPS = 120;
+    
+    private int countProactiveDrops() {
+        int dropped = 0;
+        dropped += countEvents("tunnel.dropLoadProactive", 60*1000);
+        dropped += countEvents("tunnel.dropLoad", 60*1000);
+        dropped += countEvents("tunnel.dropLoadBacklog", 60*1000);
+        dropped += countEvents("tunnel.dropLoadDelay", 60*1000);
+        return dropped;
+    }
+    private int countEvents(String stat, long period) {
+        RateStat rs = _context.statManager().getRate(stat);
+        if (rs != null) {
+            Rate r = rs.getRate(period);
+            if (r != null)
+                return (int)r.getCurrentEventCount();
+        }
+        return 0;
+    }
+    
     private void handleReq(RouterInfo nextPeerInfo, BuildMessageState state, BuildRequestRecord req, Hash nextPeer) {
         long ourId = req.readReceiveTunnelId();
         long nextId = req.readNextTunnelId();
@@ -357,17 +411,21 @@ class BuildHandler {
         //if ( (response == 0) && (_context.random().nextInt(50) <= 1) )
         //    response = TunnelHistory.TUNNEL_REJECT_PROBABALISTIC_REJECT;
         
+        int proactiveDrops = countProactiveDrops();
         long recvDelay = System.currentTimeMillis()-state.recvTime;
-        if ( (response == 0) && (recvDelay > BuildRequestor.REQUEST_TIMEOUT) ) {
-            _context.statManager().addRateData("tunnel.rejectOverloaded", recvDelay, recvDelay);
-            response = TunnelHistory.TUNNEL_REJECT_TRANSIENT_OVERLOAD;
+        if ( (response == 0) && ( (recvDelay > BuildRequestor.REQUEST_TIMEOUT/2) || (proactiveDrops > MAX_PROACTIVE_DROPS) ) ) {
+            _context.statManager().addRateData("tunnel.rejectOverloaded", recvDelay, proactiveDrops);
+            if (true || (proactiveDrops < MAX_PROACTIVE_DROPS*2))
+                response = TunnelHistory.TUNNEL_REJECT_TRANSIENT_OVERLOAD;
+            else
+                response = TunnelHistory.TUNNEL_REJECT_BANDWIDTH;
         } else if (response == 0) {
             _context.statManager().addRateData("tunnel.acceptLoad", recvDelay, recvDelay);
         }
         
         if (_log.shouldLog(Log.DEBUG))
             _log.debug("Responding to " + state.msg.getUniqueId() + "/" + ourId
-                       + " after " + recvDelay + " with " + response 
+                       + " after " + recvDelay + "/" + proactiveDrops + " with " + response 
                        + " from " + (state.fromHash != null ? state.fromHash.toBase64() : 
                                      state.from != null ? state.from.calculateHash().toBase64() : "tunnel"));
 
@@ -478,6 +536,12 @@ class BuildHandler {
         }
     }
     
+    public int getInboundBuildQueueSize() {
+        synchronized (_inboundBuildMessages) {
+            return _inboundBuildMessages.size();
+        }
+    }
+    
     private static final boolean HANDLE_REPLIES_INLINE = true;
 
     private class TunnelBuildMessageHandlerJobBuilder implements HandlerJobBuilder {
@@ -523,17 +587,31 @@ class BuildHandler {
                 } else {
                     synchronized (_inboundBuildMessages) {
                         boolean removed = false;
-                        while (_inboundBuildMessages.size() > 0) {
-                            BuildMessageState cur = (BuildMessageState)_inboundBuildMessages.get(0);
+                        int dropped = 0;
+                        for (int i = 0; i < _inboundBuildMessages.size(); i++) {
+                            BuildMessageState cur = (BuildMessageState)_inboundBuildMessages.get(i);
                             long age = System.currentTimeMillis() - cur.recvTime;
                             if (age >= BuildRequestor.REQUEST_TIMEOUT) {
-                                _inboundBuildMessages.remove(0);
+                                _inboundBuildMessages.remove(i);
+                                i--;
+                                dropped++;
                                 _context.statManager().addRateData("tunnel.dropLoad", age, _inboundBuildMessages.size());
-                            } else {
-                                break;
                             }
                         }
-                        _inboundBuildMessages.add(new BuildMessageState(receivedMessage, from, fromHash));
+                        if (dropped > 0) {
+                            // if the queue is backlogged, stop adding new messages
+                            _context.statManager().addRateData("tunnel.dropLoadBacklog", _inboundBuildMessages.size(), _inboundBuildMessages.size());
+                        } else {
+                            int queueTime = estimateQueueTime(_inboundBuildMessages.size());
+                            float pDrop = queueTime/((float)BuildRequestor.REQUEST_TIMEOUT/2);
+                            pDrop = pDrop * pDrop;
+                            float f = _context.random().nextFloat();
+                            if (pDrop > f) {
+                                _context.statManager().addRateData("tunnel.dropLoadProactive", queueTime, _inboundBuildMessages.size());
+                            } else {
+                                _inboundBuildMessages.add(new BuildMessageState(receivedMessage, from, fromHash));
+                            }
+                        }
                     }
                     _exec.repoll();
                 }
@@ -542,6 +620,28 @@ class BuildHandler {
         }
     }
     
+    private int estimateQueueTime(int numPendingMessages) {
+        int decryptTime = 200;
+        RateStat rs = _context.statManager().getRate("crypto.elGamal.decrypt");
+        if (rs != null) {
+            Rate r = rs.getRate(60*1000);
+            double avg = 0;
+            if (r != null)
+                avg = r.getAverageValue();
+            if (avg > 0) {
+                decryptTime = (int)avg;
+            } else {
+                avg = rs.getLifetimeAverageValue();
+                if (avg > 0)
+                    decryptTime = (int)avg;
+            }
+        }
+        int estimatedQueueTime = numPendingMessages * decryptTime;
+        estimatedQueueTime *= 2; // lets leave some cpu to spare, 'eh?
+        return estimatedQueueTime;
+    }
+    
+    
     private class TunnelBuildReplyMessageHandlerJobBuilder implements HandlerJobBuilder {
         public Job createJob(I2NPMessage receivedMessage, RouterIdentity from, Hash fromHash) {
             if (_log.shouldLog(Log.DEBUG))

router/java/src/net/i2p/router/tunnel/pool/ExploratoryPeerSelector.java

@@ -3,6 +3,8 @@ package net.i2p.router.tunnel.pool;
 import java.util.*;
 import net.i2p.router.RouterContext;
 import net.i2p.router.TunnelPoolSettings;
+import net.i2p.stat.Rate;
+import net.i2p.stat.RateStat;
 import net.i2p.util.Log;
 
 /**
@@ -30,7 +32,7 @@ class ExploratoryPeerSelector extends TunnelPeerSelector {
         Set exclude = getExclude(ctx, settings.isInbound(), settings.isExploratory());
         exclude.add(ctx.routerHash());
         HashSet matches = new HashSet(length);
-        boolean exploreHighCap = Boolean.valueOf(ctx.getProperty("router.exploreHighCapacity", "false")).booleanValue();
+        boolean exploreHighCap = shouldPickHighCap(ctx);
         if (exploreHighCap) 
             ctx.profileOrganizer().selectHighCapacityPeers(length, exclude, matches);
         else
@@ -48,4 +50,38 @@ class ExploratoryPeerSelector extends TunnelPeerSelector {
             rv.add(ctx.routerHash());
         return rv;
     }
+    
+    private boolean shouldPickHighCap(RouterContext ctx) {
+        if (Boolean.valueOf(ctx.getProperty("router.exploreHighCapacity", "false")).booleanValue())
+            return true;
+        // no need to explore too wildly at first
+        if (ctx.router().getUptime() <= 10*1000)
+            return true;
+        // ok, if we aren't explicitly asking for it, we should try to pick peers
+        // randomly from the 'not failing' pool.  However, if we are having a
+        // hard time building exploratory tunnels, lets fall back again on the
+        // high capacity peers, at least for a little bit.
+        int failPct = getExploratoryFailPercentage(ctx);
+        return (failPct >= ctx.random().nextInt(100));
+    }
+    
+    private int getExploratoryFailPercentage(RouterContext ctx) {
+        int timeout = getEvents(ctx, "tunnel.buildExploratoryExpire", 10*60*1000);
+        int reject = getEvents(ctx, "tunnel.buildExploratoryReject", 10*60*1000);
+        int accept = getEvents(ctx, "tunnel.buildExploratorySuccess", 10*60*1000);
+        if (accept + reject + timeout <= 0)
+            return 0;
+        double pct = (double)(reject + timeout) / (accept + reject + timeout);
+        return (int)(100 * pct);
+    }
+    
+    private int getEvents(RouterContext ctx, String stat, long period) {
+        RateStat rs = ctx.statManager().getRate(stat);
+        if (rs == null) 
+            return 0;
+        Rate r = rs.getRate(period);
+        if (r == null)
+            return 0;
+        return (int)r.getLastEventCount();
+    }
 }

router/java/src/net/i2p/router/tunnel/pool/TunnelPool.java

@@ -601,6 +601,7 @@ public class TunnelPool {
             peers.add(_context.routerHash());
         }
         PooledTunnelCreatorConfig cfg = new PooledTunnelCreatorConfig(_context, peers.size(), settings.isInbound(), settings.getDestination());
+        cfg.setTunnelPool(this);
         // peers[] is ordered endpoint first, but cfg.getPeer() is ordered gateway first
         for (int i = 0; i < peers.size(); i++) {
             int j = peers.size() - 1 - i;

router/java/src/net/i2p/router/tunnel/pool/TunnelPoolManager.java

@@ -8,10 +8,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
-import net.i2p.data.DataHelper;
-import net.i2p.data.Destination;
-import net.i2p.data.Hash;
-import net.i2p.data.TunnelId;
+import net.i2p.data.*;
 import net.i2p.data.i2np.*;
 import net.i2p.stat.RateStat;
 import net.i2p.router.*;
@@ -390,6 +387,9 @@ public class TunnelPoolManager implements TunnelManagerFacade {
     void tunnelFailed() { _executor.repoll(); }
     BuildExecutor getExecutor() { return _executor; }
     boolean isShutdown() { return _isShutdown; }
+
+    public int getInboundBuildQueueSize() { return _executor.getInboundBuildQueueSize(); }
+    
     
     public void renderStatusHTML(Writer out) throws IOException {
         out.write("<h2><a name=\"exploratory\">Exploratory tunnels</a> (<a href=\"/configtunnels.jsp#exploratory\">config</a>):</h2>\n");
@@ -499,7 +499,7 @@ public class TunnelPoolManager implements TunnelManagerFacade {
                 if (_context.routerHash().equals(peer))
                     out.write("<td><i>" + peer.toBase64().substring(0,4) + (id == null ? "" : ":" + id) + "</i></td>");
                 else
-                    out.write("<td>" + peer.toBase64().substring(0,4) + (id == null ? "" : ":" + id) + "</td>");
+                    out.write("<td>" + peer.toBase64().substring(0,4) + (id == null ? "" : ":" + id) + "</td>");                
             }
             out.write("</tr>\n");