* Watchdog:

      - Log memory stats
      - Dump threads on linux
      - Restart after 20 minutes (give the dog his teeth back)

router/java/src/net/i2p/router/RouterWatchdog.java

@@ -3,6 +3,7 @@ package net.i2p.router;
 import net.i2p.data.DataHelper;
 import net.i2p.stat.Rate;
 import net.i2p.stat.RateStat;
+import net.i2p.util.ShellCommand;
 import net.i2p.util.Log;
 
 /**
@@ -13,6 +14,7 @@ import net.i2p.util.Log;
 class RouterWatchdog implements Runnable {
     private Log _log;
     private RouterContext _context;
+    private int _consecutiveErrors;
     
     private static final long MAX_JOB_RUN_LAG = 60*1000;
     
@@ -47,6 +49,10 @@ class RouterWatchdog implements Runnable {
     }
     
     private boolean shutdownOnHang() {
+        // Client manager starts complaining after 10 minutes, and we run every minute,
+        // so this will restart 20 minutes after we lose a lease, if the wrapper is present.
+        if (_consecutiveErrors >= 10 && System.getProperty("wrapper.version") != null)
+            return true;
         return Boolean.valueOf(_context.getProperty("watchdog.haltOnHang", "false")).booleanValue();
     }
     
@@ -80,6 +86,18 @@ class RouterWatchdog implements Runnable {
                 r = rs.getRate(60*1000);
             double kbps = (r != null ? r.getAverageValue() : 0);
             _log.error("Outbound send rate: " + kbps + "KBps");
+            long max = Runtime.getRuntime().maxMemory();
+            long used = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
+            _log.error("Memory: " + DataHelper.formatSize(used) + '/' + DataHelper.formatSize(max));
+            if (_consecutiveErrors == 1) {
+                // This might work on linux...
+                // It won't on windows, and we can't call i2prouter.bat either, it does something
+                // completely different...
+                ShellCommand sc = new ShellCommand();
+                boolean success = sc.executeSilentAndWaitTimed("./i2prouter dump", 10);
+                if (success)
+                    _log.error("DUMPED THREADS TO WRAPPER LOG");
+            }
         }
     }
     
@@ -103,7 +121,10 @@ class RouterWatchdog implements Runnable {
 
         ok = ok && (verifyClientLiveliness() || netErrors >= 5);
         
-        if (!ok) {
+        if (ok) {
+            _consecutiveErrors = 0;
+        } else {
+            _consecutiveErrors++;
             dumpStatus();
             if (shutdownOnHang()) {
                 _log.log(Log.CRIT, "Router hung!  hard restart!");