0.9.6

This will allow overrides to be set without editing /usr/bin/i2prouter.

i2prouter may be replaced during upgrades. /etc/default/i2p will only be
changed by debconf (e.g. "dpkg-reconfigure i2p").

.mtn-ignore

@@ -2,6 +2,7 @@
 # Use mtn add --no-respect-ignore foo.jar to ignore this ignore list
 _jsp\.java$
 \.bz2$
+\.tar$
 \.class$
 \.diff$
 \.exe$
@@ -15,6 +16,7 @@ _jsp\.java$
 \.su2$
 \.tar$
 \.war$
+.\deb$
 \.zip$
 ^\.
 ^build
@@ -23,4 +25,7 @@ _jsp\.java$
 /build
 /classes/
 ^debian/copyright
+^debian/changelog
 override.properties
+sloccount.sc
+^reports/

.tx/config

@@ -9,7 +9,10 @@ trans.es = apps/i2ptunnel/locale/messages_es.po
 trans.fr = apps/i2ptunnel/locale/messages_fr.po
 trans.hu = apps/i2ptunnel/locale/messages_hu.po
 trans.it = apps/i2ptunnel/locale/messages_it.po
+trans.nb = apps/i2ptunnel/locale/messages_nb.po
 trans.nl = apps/i2ptunnel/locale/messages_nl.po
+trans.pl = apps/i2ptunnel/locale/messages_pl.po
+trans.pt = apps/i2ptunnel/locale/messages_pt.po
 trans.ru = apps/i2ptunnel/locale/messages_ru.po
 trans.sv_SE = apps/i2ptunnel/locale/messages_sv.po
 trans.uk_UA = apps/i2ptunnel/locale/messages_uk.po
@@ -25,13 +28,15 @@ trans.da = apps/routerconsole/locale/messages_da.po
 trans.de = apps/routerconsole/locale/messages_de.po
 trans.el = apps/routerconsole/locale/messages_el.po
 trans.es = apps/routerconsole/locale/messages_es.po
-trans.et_EE = apps/routerconsole/locale/messages_ee.po
+trans.et_EE = apps/routerconsole/locale/messages_et.po
 trans.fi = apps/routerconsole/locale/messages_fi.po
 trans.fr = apps/routerconsole/locale/messages_fr.po
 trans.hu = apps/routerconsole/locale/messages_hu.po
 trans.it = apps/routerconsole/locale/messages_it.po
+trans.nb = apps/routerconsole/locale/messages_nb.po
 trans.nl = apps/routerconsole/locale/messages_nl.po
 trans.pl = apps/routerconsole/locale/messages_pl.po
+trans.pt = apps/routerconsole/locale/messages_pt.po
 trans.ru = apps/routerconsole/locale/messages_ru.po
 trans.sv_SE = apps/routerconsole/locale/messages_sv.po
 trans.uk_UA = apps/routerconsole/locale/messages_uk.po
@@ -48,10 +53,11 @@ trans.es = apps/i2psnark/locale/messages_es.po
 trans.fr = apps/i2psnark/locale/messages_fr.po
 trans.hu = apps/i2psnark/locale/messages_hu.po
 trans.it = apps/i2psnark/locale/messages_it.po
+trans.nb = apps/i2psnark/locale/messages_nb.po
 trans.nl = apps/i2psnark/locale/messages_nl.po
 trans.pl = apps/i2psnark/locale/messages_pl.po
 trans.pt = apps/i2psnark/locale/messages_pt.po
-trans.ru = apps/i2psnark/locale/messages_ru.po
+trans.ru_RU = apps/i2psnark/locale/messages_ru.po
 trans.sv_SE = apps/i2psnark/locale/messages_sv.po
 trans.vi = apps/i2psnark/locale/messages_vi.po
 trans.zh_CN = apps/i2psnark/locale/messages_zh.po
@@ -70,6 +76,7 @@ trans.hu = apps/susidns/locale/messages_hu.po
 trans.it = apps/susidns/locale/messages_it.po
 trans.nl = apps/susidns/locale/messages_nl.po
 trans.pl = apps/susidns/locale/messages_pl.po
+trans.pt = apps/susidns/locale/messages_pt.po
 trans.ru = apps/susidns/locale/messages_ru.po
 trans.sv_SE = apps/susidns/locale/messages_sv.po
 trans.uk_UA = apps/susidns/locale/messages_uk.po
@@ -109,6 +116,7 @@ trans.nl = apps/susimail/locale/messages_nl.po
 trans.ru = apps/susimail/locale/messages_ru.po
 trans.sv_SE = apps/susimail/locale/messages_sv.po
 trans.pl = apps/susimail/locale/messages_pl.po
+trans.pt = apps/susimail/locale/messages_pt.po
 trans.uk_UA = apps/susimail/locale/messages_uk.po
 trans.vi = apps/susimail/locale/messages_vi.po
 trans.zh_CN = apps/susimail/locale/messages_zh.po
@@ -120,6 +128,7 @@ trans.cs = debian/po/cs.po
 trans.de = debian/po/de.po
 trans.el = debian/po/el.po
 trans.es = debian/po/es.po
+trans.fr = debian/po/fr.po
 trans.it = debian/po/it.po
 trans.hu = debian/po/hu.po
 trans.pl = debian/po/pl.po
@@ -127,6 +136,15 @@ trans.ru = debian/po/ru.po
 trans.sv_SE = debian/po/sv.po
 trans.uk_UA = debian/po/uk.po
 
+[I2P.i2prouter-script]
+source_file = installer/resources/locale/po/messages_en.po
+source_lang = en
+trans.de = installer/resources/locale/po/messages_de.po
+trans.fr = installer/resources/locale/po/messages_fr.po
+trans.it = installer/resources/locale/po/messages_it.po
+trans.sv_SE = installer/resources/locale/po/messages_sv.po
+trans.ru_RU = installer/resources/locale/po/messages_ru.po
+
 [main]
 host = http://www.transifex.net
 

INSTALL-headless.txt

@@ -11,7 +11,7 @@ you may use:
   lynx http://localhost:7657/
 to configure the router.
 
-If you're having trouble, swing by http://forum.i2p2.de/, check the
+If you're having trouble, swing by http://forum.i2p/, check the
 website at http://www.i2p2.de/, or get on irc://irc.freenode.net/#i2p
 
 I2P will create and store files and configuration data in the user directory

LICENSE.txt

@@ -72,6 +72,9 @@ Public domain except as listed below:
    Copyright (c) 2006, Matthew Estes
    See licenses/LICENSE-BlockFile.txt
 
+   SipHashInline:
+   Copyright 2012  Hiroshi Nakamura <nahi@ruby-lang.org>
+   See licenses/LICENSE-Apache2.0.txt
 
 Router (router.jar):
 Public domain except as listed below:
@@ -174,10 +177,11 @@ Applications:
    By welterde.
    See licenses/LICENSE-GPLv2.txt
 
-   Jetty 6.1.26:
-   Copyright 1995-2009 Mort Bay Consulting Pty Ltd
-   See licenses/LICENSE-Jetty.txt
+   Jetty 7.6.10.v20130312:
+   See licenses/ABOUT-Jetty.html
+   See licenses/NOTICE-Jetty.html
    See licenses/LICENSE-Apache2.0.txt
+   See licenses/LICENSE-ECLIPSE-1.0.html
    See licenses/NOTICE-Commons-Logging.txt
 
    JRobin 1.5.9.1:
@@ -197,6 +201,7 @@ Applications:
          - Jersey and EU flag icons: public domain, courtesy Xrmap flag
            collection http://www.arvernes.com/wiki/index.php/Xrmap
          - Guernsey and Isle of Man flags from the Open Clip Art Library, released into the public domain
+         - Curaçao, courtesy of David Benbennick, released into the public domain
          - All other flag icons: public domain, courtesy mjames@gmail.com http://www.famfamfam.com/
        Silk icons: See licenses/LICENSE-SilkIcons.txt
        FatCow icons: See licenses/LICENSE-FatCowIcons.txt
@@ -238,8 +243,8 @@ Applications:
       Bundles systray4j-2.4.1:
       See licenses/LICENSE-LGPLv2.1.txt
 
-   Tomcat 6.0.35:
-   Copyright 1999-2011 The Apache Software Foundation
+   Tomcat 6.0.36:
+   Copyright 1999-2012 The Apache Software Foundation
    See licenses/LICENSE-Apache2.0.txt
    See licenses/NOTICE-Tomcat.txt
 

README.txt

@@ -32,7 +32,7 @@ FAQ:
 
 Need help?
 	IRC irc.freenode.net #i2p
-	http://forum.i2p2.de/
+	http://forum.i2p/
 
 Licenses:
 	See LICENSE.txt

Slackware/i2p/i2p.SlackBuild

@@ -128,7 +128,7 @@ cat $CWD/slack-desc > $PKG/install/slack-desc
 
 cd $PKG
 #
-# requiredbuilder fucks up REALLY bad, and thinks java is perl?!
+# requiredbuilder messes up REALLY bad, and thinks java is perl?!
 # It also did not catch the shell requirements! BOOOOOOOOOOO! HISSSSSSSS!
 #
 # requiredbuilder -v -y -s $CWD $PKG

apps/BOB/Demos/echo/echoclient/src/net/i2p/BOB/Demos/echo/echoclient/Main.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB.Demos.echo.echoclient;
 
@@ -55,7 +47,7 @@ public class Main {
 				// exit on anything not legal
 				break;
 			}
-			c = (char)(b & 0x7f); // We only really give a fuck about ASCII
+			c = (char)(b & 0x7f); // We only care about ASCII
 			S = new String(S + c);
 		}
 		return S;

apps/BOB/Demos/echo/echoserver/src/net/i2p/BOB/Demos/echo/echoserver/Main.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB.Demos.echo.echoserver;
 
@@ -52,7 +44,7 @@ public class Main {
 			if(b < 20) {
 				break;
 			}
-			c = (char)(b & 0x7f); // We only really give a fuck about ASCII
+			c = (char)(b & 0x7f); // We only care about ASCII
 			S = new String(S + c);
 		}
 		return S;

apps/BOB/build.xml

@@ -83,7 +83,7 @@
 	        <pathelement path="${javac.classpath}" />
 	    </path>
 	</copy>
-	<copy todir="${dist.dir}/lib" file="../../installer/lib/jbigi/jbigi.jar" />
+	<copy todir="${dist.dir}/lib" file="../../build/jbigi.jar" />
 
 	<!-- Extract the classes inside the jar files -->
 	<unjar dest="${dist.dir}/classes" >

apps/BOB/nbproject/project.properties

@@ -30,11 +30,11 @@ excludes=**/*.html,**/*.txt
 file.reference.build-javadoc=../../i2p.i2p/build/javadoc
 file.reference.i2p.jar=../../core/java/build/i2p.jar
 file.reference.i2ptunnel.jar=../i2ptunnel/java/build/i2ptunnel.jar
-file.reference.jbigi.jar=../../installer/lib/jbigi/jbigi.jar
+file.reference.jbigi.jar=../../build/jbigi.jar
 file.reference.mstreaming.jar=../ministreaming/java/build/mstreaming.jar
 file.reference.router.jar=../../router/java/build/router.jar
 file.reference.streaming.jar=../streaming/java/build/streaming.jar
-file.reference.wrapper.jar=../../installer/lib/wrapper/linux/wrapper.jar
+file.reference.wrapper.jar=../../installer/lib/wrapper/all/wrapper.jar
 includes=**
 jar.compress=true
 javac.classpath=\

apps/BOB/src/net/i2p/BOB/BOB.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB;
 

apps/BOB/src/net/i2p/BOB/DoCMDS.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB;
 

apps/BOB/src/net/i2p/BOB/I2Plistener.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB;
 
@@ -94,7 +86,7 @@ public class I2Plistener implements Runnable {
 
 				}
 			} catch (I2PException e) {
-				// bad shit
+				// bad stuff
 				System.out.println("Exception " + e);
 			}
 		} finally {

apps/BOB/src/net/i2p/BOB/I2PtoTCP.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB;
 

apps/BOB/src/net/i2p/BOB/Lifted.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB;
 

apps/BOB/src/net/i2p/BOB/MUXlisten.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB;
 
@@ -302,14 +294,14 @@ public class MUXlisten implements Runnable {
 
 			// Hopefully nuke stuff here...
 			{
-				String boner = tg.getName();
+				String groupName = tg.getName();
 				try {
-					_log.warn("destroySocketManager " + boner);
+					_log.warn("destroySocketManager " + groupName);
 					socketManager.destroySocketManager();
-					_log.warn("destroySocketManager Successful" + boner);
+					_log.warn("destroySocketManager Successful" + groupName);
 				} catch (Exception e) {
 					// nop
-					_log.warn("destroySocketManager Failed" + boner);
+					_log.warn("destroySocketManager Failed" + groupName);
 					_log.warn(e.toString());
 				}
 			}
@@ -333,25 +325,25 @@ public class MUXlisten implements Runnable {
 
 			// Wait around till all threads are collected.
 			if (tg != null) {
-				String boner = tg.getName();
-				// System.out.println("BOB: MUXlisten: Starting thread collection for: " + boner);
-				_log.warn("BOB: MUXlisten: Starting thread collection for: " + boner);
+				String groupName = tg.getName();
+				// System.out.println("BOB: MUXlisten: Starting thread collection for: " + groupName);
+				_log.warn("BOB: MUXlisten: Starting thread collection for: " + groupName);
 				if (tg.activeCount() + tg.activeGroupCount() != 0) {
-					// visit(tg, 0, boner);
+					// visit(tg, 0, groupName);
 					int foo = tg.activeCount() + tg.activeGroupCount();
 					// hopefully no longer needed!
 					// int bar = lives;
-					// System.out.println("BOB: MUXlisten: Waiting on threads for " + boner);
-					// System.out.println("\nBOB: MUXlisten: ThreadGroup dump BEGIN " + boner);
-					// visit(tg, 0, boner);
-					// System.out.println("BOB: MUXlisten: ThreadGroup dump END " + boner + "\n");
+					// System.out.println("BOB: MUXlisten: Waiting on threads for " + groupName);
+					// System.out.println("\nBOB: MUXlisten: ThreadGroup dump BEGIN " + groupName);
+					// visit(tg, 0, groupName);
+					// System.out.println("BOB: MUXlisten: ThreadGroup dump END " + groupName + "\n");
 					// Happily spin forever :-(
 					while (foo != 0) {
 						foo = tg.activeCount() + tg.activeGroupCount();
 						// if (lives != bar && lives != 0) {
-						// 	System.out.println("\nBOB: MUXlisten: ThreadGroup dump BEGIN " + boner);
-						// 	visit(tg, 0, boner);
-						// 	System.out.println("BOB: MUXlisten: ThreadGroup dump END " + boner + "\n");
+						// 	System.out.println("\nBOB: MUXlisten: ThreadGroup dump BEGIN " + groupName);
+						// 	visit(tg, 0, groupName);
+						// 	System.out.println("BOB: MUXlisten: ThreadGroup dump END " + groupName + "\n");
 						// }
 						// bar = lives;
 						try {
@@ -361,8 +353,8 @@ public class MUXlisten implements Runnable {
 						}
 					}
 				}
-				// System.out.println("BOB: MUXlisten: Threads went away. Success: " + boner);
-				_log.warn("BOB: MUXlisten: Threads went away. Success: " + boner);
+				// System.out.println("BOB: MUXlisten: Threads went away. Success: " + groupName);
+				_log.warn("BOB: MUXlisten: Threads went away. Success: " + groupName);
 				tg.destroy();
 				// Zap reference to the ThreadGroup so the JVM can GC it.
 				tg = null;

apps/BOB/src/net/i2p/BOB/Main.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB;
 

apps/BOB/src/net/i2p/BOB/NamedDB.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB;
 

apps/BOB/src/net/i2p/BOB/TCPio.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB;
 

apps/BOB/src/net/i2p/BOB/TCPlistener.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB;
 

apps/BOB/src/net/i2p/BOB/TCPtoI2P.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB;
 
@@ -94,7 +86,7 @@ public class TCPtoI2P implements Runnable {
 				// exit on anything not legal
 				break;
 			}
-			c = (char) (b & 0x7f); // We only really give a fuck about ASCII
+			c = (char) (b & 0x7f); // We only care about ASCII
 			S = new String(S + c);
 		}
 		return S;

apps/BOB/src/net/i2p/BOB/UDPIOthread.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB;
 

apps/addressbook/java/src/net/i2p/addressbook/ConfigParser.java

@@ -37,6 +37,7 @@ import java.util.Map;
 
 import net.i2p.util.SecureFile;
 import net.i2p.util.SecureFileOutputStream;
+import net.i2p.util.SystemVersion;
 
 /**
  * Utility class providing methods to parse and write files in config file
@@ -49,7 +50,7 @@ import net.i2p.util.SecureFileOutputStream;
  */
 class ConfigParser {
 
-    private static final boolean isWindows = System.getProperty("os.name").startsWith("Win");
+    private static final boolean isWindows = SystemVersion.isWindows();
 
     /**
      * Strip the comments from a String. Lines that begin with '#' and ';' are

apps/addressbook/java/src/net/i2p/addressbook/Daemon.java

@@ -230,7 +230,7 @@ public class Daemon {
      */
     public static void update(Map<String, String> settings, String home) {
         File published = null;
-        boolean should_publish = Boolean.valueOf(settings.get("should_publish")).booleanValue();
+        boolean should_publish = Boolean.parseBoolean(settings.get("should_publish"));
         if (should_publish) 
             published = new File(home, settings
                 .get("published_addressbook"));
@@ -261,7 +261,7 @@ public class Daemon {
 
         // If false, add hosts via naming service; if true, write hosts.txt file directly
         // Default false
-        if (Boolean.valueOf(settings.get("update_direct")).booleanValue()) {
+        if (Boolean.parseBoolean(settings.get("update_direct"))) {
             // Direct hosts.txt access
             File routerFile = new File(home, settings.get("router_addressbook"));
             AddressBook master;

apps/addressbook/java/src/net/i2p/addressbook/Servlet.java

@@ -42,8 +42,8 @@ import javax.servlet.http.HttpServletResponse;
  */
 public class Servlet extends HttpServlet {
     private DaemonThread thread;
-    private String nonce;
-    private static final String PROP_NONCE = "addressbook.nonce";
+    //private String nonce;
+    //private static final String PROP_NONCE = "addressbook.nonce";
 
     /**
      * Hack to allow susidns to kick the daemon when the subscription list changes.
@@ -54,15 +54,15 @@ public class Servlet extends HttpServlet {
      */
     public void service(HttpServletRequest request, HttpServletResponse response) throws IOException {
         //System.err.println("Got request nonce = " + request.getParameter("nonce"));
-        if (this.thread != null && request.getParameter("wakeup") != null &&
-            this.nonce != null && this.nonce.equals(request.getParameter("nonce"))) {
-            //System.err.println("Sending interrupt");
-            this.thread.interrupt();
-            // no output
-        } else {
+        //if (this.thread != null && request.getParameter("wakeup") != null &&
+        //    this.nonce != null && this.nonce.equals(request.getParameter("nonce"))) {
+        //    //System.err.println("Sending interrupt");
+        //    this.thread.interrupt();
+        //    // no output
+        //} else {
             PrintWriter out = response.getWriter();
             out.write("I2P addressbook OK");
-        }
+        //}
     }
 
     /* (non-Javadoc)
@@ -75,9 +75,9 @@ public class Servlet extends HttpServlet {
         } catch (ServletException exp) {
             System.err.println("Addressbook init exception: " + exp);
         }
-        this.nonce = "" + Math.abs((new Random()).nextLong());
+        //this.nonce = "" + Math.abs((new Random()).nextLong());
         // put the nonce where susidns can get it
-        System.setProperty(PROP_NONCE, this.nonce);
+        //System.setProperty(PROP_NONCE, this.nonce);
         String[] args = new String[1];
         args[0] = config.getInitParameter("home");
         this.thread = new DaemonThread(args);

apps/desktopgui/.classpath

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry kind="src" path="src"/>
+	<classpathentry combineaccessrules="false" kind="src" path="/i2p_router"/>
+	<classpathentry combineaccessrules="false" kind="src" path="/i2p_sdk"/>
+	<classpathentry kind="lib" path="/lib/wrapper/all/wrapper.jar"/>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+	<classpathentry kind="output" path="build"/>
+</classpath>

apps/desktopgui/.project

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>desktopgui</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.jdt.core.javabuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.jdt.core.javanature</nature>
+	</natures>
+</projectDescription>

apps/desktopgui/build.xml

@@ -8,12 +8,15 @@
 	<property name="resources" value="resources"/>
     <property name="javadoc" value="javadoc"/>
 
+    <condition property="no.bundle">
+        <isfalse value="${require.gettext}" />
+    </condition>
 	<property name="javac.compilerargs" value=""/>
 	<property name="require.gettext" value="true" />
 
 	<target name="init">
-		<mkdir dir="${build}"/>
-        <mkdir dir="${build}/${resources}"/>
+	<mkdir dir="${build}"/>
+	<mkdir dir="${build}/${resources}"/>
         <mkdir dir="${build}/${javadoc}"/>
 		<mkdir dir="${dist}"/>
 	</target>
@@ -39,7 +42,7 @@
         </copy>
 	</target>
 
-	<target name="bundle" >
+	<target name="bundle" unless="no.bundle">
         <exec executable="sh" osfamily="unix" failifexecutionfails="true" failonerror="${require.gettext}" >
             <arg value="./bundle-messages.sh" />
         </exec>

apps/desktopgui/bundle-messages.sh

@@ -1,3 +1,4 @@
+#!/bin/sh
 #
 # Update messages_xx.po and messages_xx.class files,
 # from both java and jsp sources.

apps/fortuna/build.xml

@@ -1,106 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project basedir="." default="all" name="fortuna">
-
-	<property name="cvs.base.dir" value="java/gnu-crypto" />
-	<property name="cvs.etc.dir" value="${cvs.base.dir}/etc" />
-	<property name="cvs.lib.dir" value="${cvs.base.dir}/lib" />
-	<property name="cvs.object.dir" value="${cvs.base.dir}/classes" />
-	<property name="cvs.base.crypto.object.dir" value="${cvs.object.dir}/gnu/crypto" />
-	<property name="cvs.cipher.object.dir" value="${cvs.base.crypto.object.dir}/cipher" />
-	<property name="cvs.hash.object.dir" value="${cvs.base.crypto.object.dir}/hash" />
-	<property name="cvs.prng.object.dir" value="${cvs.base.crypto.object.dir}/prng" />
-
-	<patternset id="fortuna.files">
-		<include name="${cvs.base.crypto.object.dir}/Registry.class"/>
-		<include name="${cvs.prng.object.dir}/Fortuna*.class"/>
-		<include name="${cvs.prng.object.dir}/BasePRNG.class"/>
-		<include name="${cvs.prng.object.dir}/RandomEventListener.class"/>
-		<include name="${cvs.prng.object.dir}/IRandom.class"/>
-		<include name="${cvs.cipher.object.dir}/CipherFactory.class"/>
-		<include name="${cvs.cipher.object.dir}/IBlockCipher.class"/>
-		<include name="${cvs.hash.object.dir}/HashFactory.class"/>
-		<include name="${cvs.hash.object.dir}/IMessageDigest.class"/>
-	</patternset>
-
-	<target name="all" depends="build,jar" 
-		description="Create and test the custom Fortuna library" />
-
-	<target name="build" depends="-init,checkout"
-		description="Build the source and tests">
-		<ant dir="${cvs.base.dir}" target="jar" />
-	</target>
-
-	<target name="builddep" />
-
-	<target name="checkout" depends="-init" unless="cvs.source.available"
-		description="Check out GNU Crypto sources from CVS HEAD">
-		<cvs cvsRoot=":ext:anoncvs@savannah.gnu.org:/cvsroot/gnu-crypto"
-			cvsRsh="ssh"
-			dest="java"
-			package="gnu-crypto" />
-	</target>
-
-	<target name="clean"
-		description="Remove generated tests and object files">
-		<ant dir="${cvs.base.dir}" target="clean" />
-	</target>
-
-	<target name="cleandep" />
-
-	<target name="compile" />
-
-	<target name="distclean" depends="clean"
-		description="Remove all generated files">
-		<delete dir="build" />
-		<delete dir="jartemp" />
-		<!--
-			Annoyingly the GNU Crypto distclean task called here doesn't clean
-			*all* derived files from java/gnu-crypto/lib like it should.....
-		-->
-		<ant dir="${cvs.base.dir}" target="distclean" />
-		<!--
-			.....and so we mop up the rest ourselves.
-		-->
-		<delete dir="${cvs.lib.dir}" />
-	</target>
-
-	<target name="-init">
-		<available property="cvs.source.available" file="${cvs.base.dir}" />
-	</target>
-
-	<target name="jar" depends="build"
-		description="Create the custom Fortuna jar library">
-		<delete dir="build" />
-		<delete dir="jartemp" />
-		<mkdir dir="build" />
-		<mkdir dir="jartemp/${cvs.object.dir}" />
-		<copy todir="jartemp">
-			<fileset dir=".">
-				<patternset refid="fortuna.files" />
-			</fileset>
-		</copy>
-		<jar basedir="jartemp/${cvs.object.dir}" jarfile="build/fortuna.jar">
-			<manifest>
-				<section name="fortuna">
-					<attribute name="Implementation-Title" value="I2P Custom GNU Crypto Fortuna Library" />
-					<attribute name="Implementation-Version" value="CVS HEAD" /> 
-					<attribute name="Implementation-Vendor" value="Free Software Foundation" />
-					<attribute name="Implementation-Vendor-Id" value="FSF" />
-					<attribute name="Implementation-URL" value="http://www.gnu.org/software/gnu-crypto" />
-				</section>
-			</manifest>
-		</jar>
-		<delete dir="jartemp" />
-	</target>
-
-	<target name="test" depends="jar"
-		description="Perform crypto tests on custom Fortuna jar library" />
-		<!--
-			Add this when Fortuna tests are added to GNU Crypto, else write some
-		-->
-
-	<target name="update" depends="checkout"
-		description="Update GNU Crypto sources to latest CVS HEAD">
-		<cvs command="update -d" cvsRsh="ssh" dest="java/gnu-crypto" />
-	</target>
-</project>

apps/i2psnark/.classpath

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry kind="src" path="java/src"/>
+	<classpathentry combineaccessrules="false" kind="src" path="/i2p_sdk"/>
+	<classpathentry combineaccessrules="false" kind="src" path="/ministreaming"/>
+	<classpathentry kind="lib" path="/jetty/jettylib/javax.servlet.jar"/>
+	<classpathentry kind="lib" path="/jetty/jettylib/jetty-util.jar"/>
+	<classpathentry kind="lib" path="/jetty/jettylib/org.mortbay.jetty.jar"/>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+	<classpathentry kind="output" path="java/build/obj"/>
+</classpath>

apps/i2psnark/.project

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>i2psnark</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.jdt.core.javabuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.jdt.core.javanature</nature>
+	</natures>
+</projectDescription>

apps/i2psnark/java/build.xml

@@ -19,11 +19,15 @@
                 <pathelement location="../../ministreaming/java/build/obj" />
                 <pathelement location="../../jetty/jettylib/org.mortbay.jetty.jar" />
                 <pathelement location="../../jetty/jettylib/javax.servlet.jar" />
+                <pathelement location="../../jetty/jettylib/jetty-servlet.jar" />
                 <pathelement location="../../jetty/jettylib/jetty-util.jar" />
             </classpath>
         </depend>
     </target>
 
+    <condition property="no.bundle">
+        <isfalse value="${require.gettext}" />
+    </condition>
     <property name="javac.compilerargs" value="" />
     <property name="require.gettext" value="true" />
 
@@ -35,7 +39,7 @@
             debug="true" deprecation="on" source="1.5" target="1.5" 
             destdir="./build/obj" 
             includeAntRuntime="false"
-            classpath="../../../core/java/build/i2p.jar:../../jetty/jettylib/org.mortbay.jetty.jar:../../jetty/jettylib/javax.servlet.jar:../../jetty/jettylib/jetty-util.jar:../../ministreaming/java/build/mstreaming.jar" >
+            classpath="../../../core/java/build/i2p.jar:../../jetty/jettylib/org.mortbay.jetty.jar:../../jetty/jettylib/javax.servlet.jar:../../jetty/jettylib/jetty-servlet.jar:../../jetty/jettylib/jetty-util.jar:../../ministreaming/java/build/mstreaming.jar" >
             <compilerarg line="${javac.compilerargs}" />
         </javac>
     </target>
@@ -57,7 +61,7 @@
     <target name="jar" depends="builddep, compile, jarUpToDate, listChangedFiles" unless="jar.uptodate" >
         <!-- set if unset -->
         <property name="workspace.changes.tr" value="" />
-        <jar destfile="./build/i2psnark.jar" basedir="./build/obj" includes="**/*.class" excludes="**/I2PSnarkServlet*.class **/FetchAndAdd*.class **/messages_*.class">
+        <jar destfile="./build/i2psnark.jar" basedir="./build/obj" includes="**/*.class" excludes="**/web/* **/messages_*.class">
             <manifest>
                 <attribute name="Main-Class" value="org.klomp.snark.Snark" />
                 <attribute name="Class-Path" value="i2p.jar mstreaming.jar streaming.jar" />
@@ -72,7 +76,7 @@
     
     <target name="jarUpToDate">
         <uptodate property="jar.uptodate" targetfile="build/i2psnark.jar" >
-            <srcfiles dir= "build/obj" includes="**/*.class" excludes="**/I2PSnarkServlet*.class **/FetchAndAdd*.class **/messages_*.class" />
+            <srcfiles dir= "build/obj" includes="**/*.class" excludes="**/web/* **/messages_*.class" />
         </uptodate>
         <condition property="shouldListChanges" >
             <and>
@@ -80,7 +84,7 @@
                     <isset property="jar.uptodate" />
                 </not>
                 <not>
-                    <isset property="wjar.uptodate" />
+                    <isset property="war.uptodate" />
                 </not>
                 <isset property="mtn.available" />
             </and>
@@ -100,9 +104,11 @@
         <copy todir="build/icons/.icons" >
             <fileset dir="../icons/" />
         </copy>
+        <!-- mime.properties must be in with the classes -->
+        <copy file="../mime.properties" todir="build/obj/org/klomp/snark/web" />
         <war destfile="../i2psnark.war" webxml="../web.xml" >
           <!-- include only the web stuff, as of 0.7.12 the router will add i2psnark.jar to the classpath for the war -->
-          <classes dir="./build/obj" includes="**/web/*.class" />
+          <classes dir="./build/obj" includes="**/web/*" />
             <fileset dir="build/icons/" />
             <manifest>
                 <attribute name="Implementation-Version" value="${full.version}" />
@@ -120,7 +126,7 @@
         </uptodate>
     </target>
     
-    <target name="bundle" depends="compile">
+    <target name="bundle" depends="compile" unless="no.bundle">
         <!-- Update the messages_*.po files.
              We need to supply the bat file for windows, and then change the fail property to true -->
         <exec executable="sh" osfamily="unix" failifexecutionfails="true" failonerror="${require.gettext}" >

apps/i2psnark/java/bundle-messages.sh

@@ -1,3 +1,4 @@
+#!/bin/sh
 #
 # Update messages_xx.po and messages_xx.class files,
 # from both java and jsp sources.

apps/i2psnark/java/src/net/i2p/kademlia/KBucketSet.java

@@ -14,7 +14,6 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashSet;
-import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -24,6 +23,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
 import net.i2p.I2PAppContext;
 import net.i2p.data.DataHelper;
 import net.i2p.data.SimpleDataStructure;
+import net.i2p.util.LHMCache;
 import net.i2p.util.Log;
 
 /**
@@ -209,10 +209,10 @@ public class KBucketSet<T extends SimpleDataStructure> {
             int s1, e1, s2, e2;
             s1 = b0.getRangeBegin();
             e2 = b0.getRangeEnd();
-            if (B_FACTOR > 1 &&
-                (s1 & (B_FACTOR - 1)) == 0 &&
-                ((e2 + 1) & (B_FACTOR - 1)) == 0 &&
-                e2 > s1 + B_FACTOR) {
+            if (B_VALUE == 1 ||
+                ((s1 & (B_FACTOR - 1)) == 0 &&
+                 ((e2 + 1) & (B_FACTOR - 1)) == 0 &&
+                 e2 > s1 + B_FACTOR)) {
                 // The bucket is a "whole" kbucket with a range > B_FACTOR,
                 // so it should be split into two "whole" kbuckets each with
                 // a range >= B_FACTOR.
@@ -518,6 +518,7 @@ public class KBucketSet<T extends SimpleDataStructure> {
     
     /**
      *  For every bucket that hasn't been updated in this long,
+     *  or isn't close to full,
      *  generate a random key that would be a member of that bucket.
      *  The returned keys may be searched for to "refresh" the buckets.
      *  @return non-null, closest first
@@ -528,7 +529,10 @@ public class KBucketSet<T extends SimpleDataStructure> {
         getReadLock();
         try {
             for (KBucket b : _buckets) {
-                if (b.getLastChanged() < old)
+                int curSize = b.getKeyCount();
+                // Always explore the closest bucket
+                if ((b.getRangeBegin() == 0) ||
+                    (b.getLastChanged() < old || curSize < BUCKET_SIZE * 3 / 4))
                     rv.add(generateRandomKey(b));
             }
         } finally { releaseReadLock(); }
@@ -658,7 +662,7 @@ public class KBucketSet<T extends SimpleDataStructure> {
         public Range(T us, int bValue) {
             _bValue = bValue;
             _bigUs = new BigInteger(1, us.getData());
-            _distanceCache = new LHM(256);
+            _distanceCache = new LHMCache(256);
         }
 
         /** @return 0 to max-1 or -1 for us */
@@ -697,20 +701,6 @@ public class KBucketSet<T extends SimpleDataStructure> {
         }
     }
 
-    private static class LHM<K, V> extends LinkedHashMap<K, V> {
-        private final int _max;
-
-        public LHM(int max) {
-            super(max, 0.75f, true);
-            _max = max;
-        }
-
-        @Override
-        protected boolean removeEldestEntry(Map.Entry<K, V> eldest) {
-            return size() > _max;
-        }
-    }
-
     /**
      *  For Collections.binarySearch.
      *  getRangeBegin == getRangeEnd.
@@ -772,8 +762,8 @@ public class KBucketSet<T extends SimpleDataStructure> {
     public String toString() {
         StringBuilder buf = new StringBuilder(1024);
         buf.append("Bucket set rooted on: ").append(_us.toString())
-           .append(" K= ").append(BUCKET_SIZE)
-           .append(" B= ").append(B_VALUE)
+           .append(" K=").append(BUCKET_SIZE)
+           .append(" B=").append(B_VALUE)
            .append(" with ").append(size())
            .append(" keys in ").append(_buckets.size()).append(" buckets:\n");
         getReadLock();

apps/i2psnark/java/src/org/klomp/snark/BitField.java

@@ -39,7 +39,6 @@ public class BitField
     this.size = size;
     int arraysize = ((size-1)/8)+1;
     bitfield = new byte[arraysize];
-    this.count = 0;
   }
 
   /**
@@ -60,7 +59,6 @@ public class BitField
     // cleared or clear them explicitly ourselves.
     System.arraycopy(bitfield, 0, this.bitfield, 0, arraysize);
 
-    this.count = 0;
     for (int i = 0; i < size; i++)
       if (get(i))
         this.count++;
@@ -99,9 +97,11 @@ public class BitField
       throw new IndexOutOfBoundsException(Integer.toString(bit));
     int index = bit/8;
     int mask = 128 >> (bit % 8);
-    if ((bitfield[index] & mask) == 0) {
-      count++;
-      bitfield[index] |= mask;
+    synchronized(this) {
+        if ((bitfield[index] & mask) == 0) {
+            count++;
+            bitfield[index] |= mask;
+        }
     }
   }
 

apps/i2psnark/java/src/org/klomp/snark/CompleteListener.java

@@ -0,0 +1,60 @@
+/* CompleteListener - Callback for Snark events
+   
+   Copyright (C) 2003 Mark J. Wielaard
+
+   This file is part of Snark.
+   
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+ 
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+ 
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software Foundation,
+   Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+*/
+
+package org.klomp.snark;
+
+/**
+ *  Callback for Snark events.
+ *  @since 0.9.4 moved from Snark.java
+ */
+public interface CompleteListener {
+    public void torrentComplete(Snark snark);
+    public void updateStatus(Snark snark);
+
+    /**
+     * We transitioned from magnet mode, we have now initialized our
+     * metainfo and storage. The listener should now call getMetaInfo()
+     * and save the data to disk.
+     *
+     * @return the new name for the torrent or null on error
+     * @since 0.8.4
+     */
+    public String gotMetaInfo(Snark snark);
+
+    /**
+     * @since 0.9
+     */
+    public void fatal(Snark snark, String error);
+
+    /**
+     * @since 0.9.2
+     */
+    public void addMessage(Snark snark, String message);
+
+    /**
+     * @since 0.9.4
+     */
+    public void gotPiece(Snark snark);
+
+    // not really listeners but the easiest way to get back to an optional SnarkManager
+    public long getSavedTorrentTime(Snark snark);
+    public BitField getSavedTorrentBitField(Snark snark);
+}

apps/i2psnark/java/src/org/klomp/snark/ConnectionAcceptor.java

@@ -39,7 +39,7 @@ import net.i2p.util.SimpleTimer;
 /**
  * Accepts connections on a TCP port and routes them to sub-acceptors.
  */
-public class ConnectionAcceptor implements Runnable
+class ConnectionAcceptor implements Runnable
 {
   private final Log _log = I2PAppContext.getGlobalContext().logManager().getLog(ConnectionAcceptor.class);
   private I2PServerSocket serverSocket;

apps/i2psnark/java/src/org/klomp/snark/DataLoader.java

@@ -1,5 +1,7 @@
 package org.klomp.snark;
 
+import net.i2p.data.ByteArray;
+
 /**
  * Callback used to fetch data
  * @since 0.8.2
@@ -10,5 +12,5 @@ interface DataLoader
    *  This is the callback that PeerConnectionOut calls to get the data from disk
    *  @return bytes or null for errors
    */
-    public byte[] loadData(int piece, int begin, int length);
+    public ByteArray loadData(int piece, int begin, int length);
 }

apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java

@@ -1,5 +1,6 @@
 package org.klomp.snark;
 
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -48,6 +49,7 @@ import org.klomp.snark.dht.KRPC;
 public class I2PSnarkUtil {
     private final I2PAppContext _context;
     private final Log _log;
+    private final String _baseName;
     
     private boolean _shouldProxy;
     private String _proxyHost;
@@ -58,7 +60,7 @@ public class I2PSnarkUtil {
     private volatile I2PSocketManager _manager;
     private boolean _configured;
     private volatile boolean _connecting;
-    private final Set<Hash> _shitlist;
+    private final Set<Hash> _banlist;
     private int _maxUploaders;
     private int _maxUpBW;
     private int _maxConnections;
@@ -78,15 +80,24 @@ public class I2PSnarkUtil {
     public static final int DEFAULT_MAX_UP_BW = 8;  //KBps
     public static final int MAX_CONNECTIONS = 16; // per torrent
     public static final String PROP_MAX_BW = "i2cp.outboundBytesPerSecond";
-    public static final boolean DEFAULT_USE_DHT = false;
+    public static final boolean DEFAULT_USE_DHT = true;
 
     public I2PSnarkUtil(I2PAppContext ctx) {
+        this(ctx, "i2psnark");
+    }
+
+    /**
+     *  @param baseName generally "i2psnark"
+     *  @since Jetty 7
+     */
+    public I2PSnarkUtil(I2PAppContext ctx, String baseName) {
         _context = ctx;
         _log = _context.logManager().getLog(Snark.class);
+        _baseName = baseName;
         _opts = new HashMap();
         //setProxy("127.0.0.1", 4444);
         setI2CPConfig("127.0.0.1", 7654, null);
-        _shitlist = new ConcurrentHashSet();
+        _banlist = new ConcurrentHashSet();
         _maxUploaders = Snark.MAX_TOTAL_UPLOADERS;
         _maxUpBW = DEFAULT_MAX_UP_BW;
         _maxConnections = MAX_CONNECTIONS;
@@ -98,7 +109,7 @@ public class I2PSnarkUtil {
         // This is used for both announce replies and .torrent file downloads,
         // so it must be available even if not connected to I2CP.
         // so much for multiple instances
-        _tmpDir = new SecureDirectory(ctx.getTempDir(), "i2psnark");
+        _tmpDir = new SecureDirectory(ctx.getTempDir(), baseName);
         FileUtil.rmdir(_tmpDir, false);
         _tmpDir.mkdirs();
     }
@@ -215,9 +226,11 @@ public class I2PSnarkUtil {
                 }
             }
             if (opts.getProperty("inbound.nickname") == null)
-                opts.setProperty("inbound.nickname", "I2PSnark");
+                opts.setProperty("inbound.nickname", _baseName.replace("i2psnark", "I2PSnark"));
             if (opts.getProperty("outbound.nickname") == null)
-                opts.setProperty("outbound.nickname", "I2PSnark");
+                opts.setProperty("outbound.nickname", _baseName.replace("i2psnark", "I2PSnark"));
+            if (opts.getProperty("outbound.priority") == null)
+                opts.setProperty("outbound.priority", "-10");
             // Dont do this for now, it is set in I2PSocketEepGet for announces,
             // we don't need fast handshake for peer connections.
             //if (opts.getProperty("i2p.streaming.connectDelay") == null)
@@ -244,11 +257,13 @@ public class I2PSnarkUtil {
                 opts.setProperty("i2p.streaming.maxConnsPerHour", "20");
             if (opts.getProperty("i2p.streaming.enforceProtocol") == null)
                 opts.setProperty("i2p.streaming.enforceProtocol", "true");
+            if (opts.getProperty("i2p.streaming.disableRejectLogging") == null)
+                opts.setProperty("i2p.streaming.disableRejectLogging", "true");
             _manager = I2PSocketManagerFactory.createManager(_i2cpHost, _i2cpPort, opts);
             _connecting = false;
         }
         if (_shouldUseDHT && _manager != null && _dht == null)
-            _dht = new KRPC(_context, _manager.getSession());
+            _dht = new KRPC(_context, _baseName, _manager.getSession());
         return (_manager != null);
     }
     
@@ -283,7 +298,7 @@ public class I2PSnarkUtil {
         I2PSocketManager mgr = _manager;
         // FIXME this can cause race NPEs elsewhere
         _manager = null;
-        _shitlist.clear();
+        _banlist.clear();
         if (mgr != null) {
             if (_log.shouldLog(Log.DEBUG))
                 _log.debug("Disconnecting from I2P", new Exception("I did it"));
@@ -306,24 +321,24 @@ public class I2PSnarkUtil {
         if (addr.equals(getMyDestination()))
             throw new IOException("Attempt to connect to myself");
         Hash dest = addr.calculateHash();
-        if (_shitlist.contains(dest))
-            throw new IOException("Not trying to contact " + dest.toBase64() + ", as they are shitlisted");
+        if (_banlist.contains(dest))
+            throw new IOException("Not trying to contact " + dest.toBase64() + ", as they are banlisted");
         try {
             I2PSocket rv = _manager.connect(addr);
             if (rv != null)
-                _shitlist.remove(dest);
+                _banlist.remove(dest);
             return rv;
         } catch (I2PException ie) {
-            _shitlist.add(dest);
-            _context.simpleScheduler().addEvent(new Unshitlist(dest), 10*60*1000);
+            _banlist.add(dest);
+            _context.simpleScheduler().addEvent(new Unbanlist(dest), 10*60*1000);
             throw new IOException("Unable to reach the peer " + peer + ": " + ie.getMessage());
         }
     }
     
-    private class Unshitlist implements SimpleTimer.TimedEvent {
+    private class Unbanlist implements SimpleTimer.TimedEvent {
         private Hash _dest;
-        public Unshitlist(Hash dest) { _dest = dest; }
-        public void timeReached() { _shitlist.remove(_dest); }
+        public Unbanlist(Hash dest) { _dest = dest; }
+        public void timeReached() { _banlist.remove(_dest); }
     }
     
     /**
@@ -391,6 +406,46 @@ public class I2PSnarkUtil {
         }
     }
     
+    /**
+     * Fetch to memory
+     * @param retries if < 0, set timeout to a few seconds
+     * @param initialSize buffer size
+     * @param maxSize fails if greater
+     * @return null on error
+     * @since 0.9.4
+     */
+    public byte[] get(String url, boolean rewrite, int retries, int initialSize, int maxSize) {
+        if (_log.shouldLog(Log.DEBUG))
+            _log.debug("Fetching [" + url + "] to memory");
+        String fetchURL = url;
+        if (rewrite)
+            fetchURL = rewriteAnnounce(url);
+        int timeout;
+        if (retries < 0) {
+            if (!connected())
+                return null;
+            timeout = EEPGET_CONNECT_TIMEOUT_SHORT;
+            retries = 0;
+        } else {
+            timeout = EEPGET_CONNECT_TIMEOUT;
+            if (!connected()) {
+                if (!connect())
+                    return null;
+            }
+        }
+        ByteArrayOutputStream out = new ByteArrayOutputStream(initialSize);
+        EepGet get = new I2PSocketEepGet(_context, _manager, retries, -1, maxSize, null, out, fetchURL);
+        if (get.fetch(timeout)) {
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("Fetch successful [" + url + "]: size=" + out.size());
+            return out.toByteArray();
+        } else {
+            if (_log.shouldLog(Log.WARN))
+                _log.warn("Fetch failed [" + url + "]");
+            return null;
+        }
+    }
+    
     public I2PServerSocket getServerSocket() { 
         I2PSocketManager mgr = _manager;
         if (mgr != null)
@@ -521,6 +576,15 @@ public class I2PSnarkUtil {
             return Collections.EMPTY_LIST;
         return _openTrackers;
     }
+
+    /**
+     *  List of open trackers to use as backups even if disabled
+     *  @return non-null
+     *  @since 0.9.4
+     */
+    public List<String> getBackupTrackers() { 
+        return _openTrackers;
+    }
     
     public void setUseOpenTrackers(boolean yes) {
         _shouldUseOT = yes;
@@ -534,7 +598,7 @@ public class I2PSnarkUtil {
     public synchronized void setUseDHT(boolean yes) {
         _shouldUseDHT = yes;
         if (yes && _manager != null && _dht == null) {
-            _dht = new KRPC(_context, _manager.getSession());
+            _dht = new KRPC(_context, _baseName, _manager.getSession());
         } else if (!yes && _dht != null) {
             _dht.stop();
             _dht = null;

apps/i2psnark/java/src/org/klomp/snark/MagnetURI.java

@@ -0,0 +1,212 @@
+package org.klomp.snark;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+import net.i2p.data.Base32;
+
+/**
+ *
+ * @since 0.9.4 moved from I2PSnarkServlet
+ */
+public class MagnetURI {
+
+    private final String _tracker;
+    private final String _name;
+    private final byte[] _ih;
+
+    /** BEP 9 */
+    public static final String MAGNET = "magnet:";
+    public static final String MAGNET_FULL = MAGNET + "?xt=urn:btih:";
+    /** http://sponge.i2p/files/maggotspec.txt */
+    public static final String MAGGOT = "maggot://";
+
+    /**
+     *  @param url non-null
+     */
+    public MagnetURI(I2PSnarkUtil util, String url) throws IllegalArgumentException {
+        String ihash;
+        String name;
+        String trackerURL = null;
+        if (url.startsWith(MAGNET)) {
+            // magnet:?xt=urn:btih:0691e40aae02e552cfcb57af1dca56214680c0c5&tr=http://tracker2.postman.i2p/announce.php
+            String xt = getParam("xt", url);
+            if (xt == null || !xt.startsWith("urn:btih:"))
+                throw new IllegalArgumentException();
+            ihash = xt.substring("urn:btih:".length());
+            trackerURL = getTrackerParam(url);
+            name = util.getString("Magnet") + ' ' + ihash;
+            String dn = getParam("dn", url);
+            if (dn != null)
+                name += " (" + Storage.filterName(dn) + ')';
+        } else if (url.startsWith(MAGGOT)) {
+            // maggot://0691e40aae02e552cfcb57af1dca56214680c0c5:0b557bbdf8718e95d352fbe994dec3a383e2ede7
+            ihash = url.substring(MAGGOT.length()).trim();
+            int col = ihash.indexOf(':');
+            if (col >= 0)
+                ihash = ihash.substring(0, col);
+            name = util.getString("Magnet") + ' ' + ihash;
+        } else {
+            throw new IllegalArgumentException();
+        }
+        byte[] ih = null;
+        if (ihash.length() == 32) {
+            ih = Base32.decode(ihash);
+        } else if (ihash.length() == 40) {
+            //  Like DataHelper.fromHexString() but ensures no loss of leading zero bytes
+            ih = new byte[20];
+            try {
+                for (int i = 0; i < 20; i++) {
+                    ih[i] = (byte) (Integer.parseInt(ihash.substring(i*2, (i*2) + 2), 16) & 0xff);
+                }
+            } catch (NumberFormatException nfe) {
+                ih = null;
+            }
+        }
+        if (ih == null || ih.length != 20)
+            throw new IllegalArgumentException();
+        _ih = ih;
+        _name = name;
+        _tracker = trackerURL;
+    }
+
+    /**
+     *  @return 20 bytes or null
+     */
+    public byte[] getInfoHash() {
+        return _ih;
+    }
+
+    /**
+     *  @return pretty name or null
+     */
+    public String getName() {
+        return _name;
+    }
+
+    /**
+     *  @return tracker url or null
+     */
+    public String getTrackerURL() {
+        return _tracker;
+    }
+
+    /**
+     *  @return first decoded parameter or null
+     */
+    private static String getParam(String key, String uri) {
+        int idx = uri.indexOf('?' + key + '=');
+        if (idx >= 0) {
+            idx += key.length() + 2;
+        } else {
+            idx = uri.indexOf('&' + key + '=');
+            if (idx >= 0)
+                idx += key.length() + 2;
+        }
+        if (idx < 0 || idx > uri.length())
+            return null;
+        String rv = uri.substring(idx);
+        idx = rv.indexOf('&');
+        if (idx >= 0)
+            rv = rv.substring(0, idx);
+        else
+            rv = rv.trim();
+        return decode(rv);
+    }
+
+    /**
+     *  @return all decoded parameters or null
+     *  @since 0.9.1
+     */
+    private static List<String> getMultiParam(String key, String uri) {
+        int idx = uri.indexOf('?' + key + '=');
+        if (idx >= 0) {
+            idx += key.length() + 2;
+        } else {
+            idx = uri.indexOf('&' + key + '=');
+            if (idx >= 0)
+                idx += key.length() + 2;
+        }
+        if (idx < 0 || idx > uri.length())
+            return null;
+        List<String> rv = new ArrayList();
+        while (true) {
+            String p = uri.substring(idx);
+            uri = p;
+            idx = p.indexOf('&');
+            if (idx >= 0)
+                p = p.substring(0, idx);
+            else
+                p = p.trim();
+            rv.add(decode(p));
+            idx = uri.indexOf('&' + key + '=');
+            if (idx < 0)
+                break;
+            idx += key.length() + 2;
+        }
+        return rv;
+    }
+
+    /**
+     *  @return first valid I2P tracker or null
+     *  @since 0.9.1
+     */
+    private static String getTrackerParam(String uri) {
+        List<String> trackers = getMultiParam("tr", uri);
+        if (trackers == null)
+            return null;
+        for (String t : trackers) {
+            try {
+                URI u = new URI(t);
+                String protocol = u.getScheme();
+                String host = u.getHost();
+                if (protocol == null || host == null ||
+                    !protocol.toLowerCase(Locale.US).equals("http") ||
+                    !host.toLowerCase(Locale.US).endsWith(".i2p"))
+                    continue;
+                return t;
+            } catch(URISyntaxException use) {}
+        }
+        return null;
+    }
+
+    /**
+     *  Decode %xx encoding, convert to UTF-8 if necessary
+     *  Copied from i2ptunnel LocalHTTPServer
+     *  @since 0.9.1
+     */
+    private static String decode(String s) {
+        if (!s.contains("%"))
+            return s;
+        StringBuilder buf = new StringBuilder(s.length());
+        boolean utf8 = false;
+        for (int i = 0; i < s.length(); i++) {
+            char c = s.charAt(i);
+            if (c != '%') {
+                buf.append(c);
+            } else {
+                try {
+                    int val = Integer.parseInt(s.substring(++i, (++i) + 1), 16);
+                    if ((val & 0x80) != 0)
+                        utf8 = true;
+                    buf.append((char) val);
+                } catch (IndexOutOfBoundsException ioobe) {
+                    break;
+                } catch (NumberFormatException nfe) {
+                    break;
+                }
+            }
+        }
+        if (utf8) {
+            try {
+                return new String(buf.toString().getBytes("ISO-8859-1"), "UTF-8");
+            } catch (UnsupportedEncodingException uee) {}
+        }
+        return buf.toString();
+    }
+
+}

apps/i2psnark/java/src/org/klomp/snark/Message.java

@@ -23,8 +23,13 @@ package org.klomp.snark;
 import java.io.DataOutputStream;
 import java.io.IOException;
 
-// Used to queue outgoing connections
-// sendMessage() should be used to translate them to wire format.
+import net.i2p.data.ByteArray;
+import net.i2p.util.ByteCache;
+
+/**
+ * Used to queue outgoing connections
+ * sendMessage() should be used to translate them to wire format.
+ */
 class Message
 {
   final static byte KEEP_ALIVE   = -1;
@@ -69,6 +74,9 @@ class Message
   // now unused
   //SimpleTimer.TimedEvent expireEvent;
   
+  private static final int BUFSIZE = PeerState.PARTSIZE;
+  private static final ByteCache _cache = ByteCache.getInstance(16, BUFSIZE);
+
   /** Utility method for sending a message through a DataStream. */
   void sendMessage(DataOutputStream dos) throws IOException
   {
@@ -79,11 +87,15 @@ class Message
         return;
       }
 
+    ByteArray ba;
     // Get deferred data
     if (data == null && dataLoader != null) {
-        data = dataLoader.loadData(piece, begin, length);
-        if (data == null)
+        ba = dataLoader.loadData(piece, begin, length);
+        if (ba == null)
             return;  // hmm will get retried, but shouldn't happen
+        data = ba.getData();
+    } else {
+        ba = null;
     }
 
     // Calculate the total length in bytes
@@ -139,6 +151,10 @@ class Message
     // Send actual data
     if (type == BITFIELD || type == PIECE || type == EXTENSION)
       dos.write(data, off, len);
+
+    // Was pulled from cache in Storage.getPiece() via dataLoader
+    if (ba != null && ba.getData().length == BUFSIZE)
+        _cache.release(ba, false);
   }
 
     @Override

apps/i2psnark/java/src/org/klomp/snark/MetaInfo.java

@@ -61,6 +61,7 @@ public class MetaInfo
   private final byte[] piece_hashes;
   private final long length;
   private final boolean privateTorrent;
+  private final List<List<String>> announce_list;
   private Map<String, BEValue> infoMap;
 
   /**
@@ -69,9 +70,11 @@ public class MetaInfo
    *  @param announce may be null
    *  @param files null for single-file torrent
    *  @param lengths null for single-file torrent
+   *  @param announce_list may be null
    */
   MetaInfo(String announce, String name, String name_utf8, List<List<String>> files, List<Long> lengths,
-           int piece_length, byte[] piece_hashes, long length, boolean privateTorrent)
+           int piece_length, byte[] piece_hashes, long length, boolean privateTorrent,
+           List<List<String>> announce_list)
   {
     this.announce = announce;
     this.name = name;
@@ -83,6 +86,7 @@ public class MetaInfo
     this.piece_hashes = piece_hashes;
     this.length = length;
     this.privateTorrent = privateTorrent;
+    this.announce_list = announce_list;
 
     // TODO if we add a parameter for other keys
     //if (other != null) {
@@ -141,6 +145,23 @@ public class MetaInfo
         this.announce = val.getString();
     }
 
+    // BEP 12
+    val = m.get("announce-list");
+    if (val == null) {
+        this.announce_list = null;
+    } else {
+        this.announce_list = new ArrayList();
+        List<BEValue> bl1 = val.getList();
+        for (BEValue bev : bl1) {
+            List<BEValue> bl2 = bev.getList();
+            List<String> sl2 = new ArrayList();           
+            for (BEValue bev2 : bl2) {
+                sl2.add(bev2.getString());
+            }
+            this.announce_list.add(sl2);
+        }
+    }
+
     val = m.get("info");
     if (val == null)
         throw new InvalidBEncodingException("Missing info map");
@@ -296,6 +317,15 @@ public class MetaInfo
     return announce;
   }
 
+  /**
+   * Returns a list of lists of urls.
+   *
+   * @since 0.9.5
+   */
+  public List<List<String>> getAnnounceList() {
+    return announce_list;
+  }
+
   /**
    * Returns the original 20 byte SHA1 hash over the bencoded info map.
    */
@@ -470,12 +500,13 @@ public class MetaInfo
   /**
    * Creates a copy of this MetaInfo that shares everything except the
    * announce URL.
+   * Drops any announce-list.
    */
   public MetaInfo reannounce(String announce)
   {
     return new MetaInfo(announce, name, name_utf8, files,
                         lengths, piece_length,
-                        piece_hashes, length, privateTorrent);
+                        piece_hashes, length, privateTorrent, null);
   }
 
   /**
@@ -486,6 +517,8 @@ public class MetaInfo
         Map m = new HashMap();
         if (announce != null)
             m.put("announce", announce);
+        if (announce_list != null)
+            m.put("announce-list", announce_list);
         Map info = createInfoMap();
         m.put("info", info);
         // don't save this locally, we should only do this once

apps/i2psnark/java/src/org/klomp/snark/PartialPiece.java

@@ -9,6 +9,8 @@ import java.security.MessageDigest;
 
 import net.i2p.I2PAppContext;
 import net.i2p.crypto.SHA1;
+import net.i2p.data.ByteArray;
+import net.i2p.util.ByteCache;
 import net.i2p.util.Log;
 import net.i2p.util.SecureFile;
 
@@ -42,6 +44,9 @@ class PartialPiece implements Comparable {
     private final int pclen;
     private final File tempDir;
 
+    private static final int BUFSIZE = PeerState.PARTSIZE;
+    private static final ByteCache _cache = ByteCache.getInstance(16, BUFSIZE);
+
     // Any bigger than this, use temp file instead of heap
     private static final int MAX_IN_MEM = 128 * 1024;
     // May be reduced on OOM
@@ -154,7 +159,16 @@ class PartialPiece implements Comparable {
             sha1.update(bs);
         } else {
             int read = 0;
-            byte[] buf = new byte[Math.min(pclen, 16384)];
+            int buflen = Math.min(pclen, BUFSIZE);
+            ByteArray ba;
+            byte[] buf;
+            if (buflen == BUFSIZE) {
+                ba = _cache.acquire();
+                buf = ba.getData();
+            } else {
+                ba = null;
+                buf = new byte[buflen];
+            }
             synchronized (this) {
                 if (raf == null)
                     throw new IOException();
@@ -167,6 +181,8 @@ class PartialPiece implements Comparable {
                     sha1.update(buf, 0, rd);
                 }
             }
+            if (ba != null)
+                _cache.release(ba, false);
             if (read < pclen)
                 throw new IOException();
         }
@@ -182,7 +198,15 @@ class PartialPiece implements Comparable {
             din.readFully(bs, off, len);
         } else {
             // read in fully before synching on raf
-            byte[] tmp = new byte[len];
+            ByteArray ba;
+            byte[] tmp;
+            if (len == BUFSIZE) {
+                ba = _cache.acquire();
+                tmp = ba.getData();
+            } else {
+                ba = null;
+                tmp = new byte[len];
+            }
             din.readFully(tmp);
             synchronized (this) {
                 if (raf == null)
@@ -190,6 +214,8 @@ class PartialPiece implements Comparable {
                 raf.seek(off);
                 raf.write(tmp);
             }
+            if (ba != null)
+                _cache.release(ba, false);
         }
     }
 
@@ -208,7 +234,16 @@ class PartialPiece implements Comparable {
             out.write(bs, offset, len);
         } else {
             int read = 0;
-            byte[] buf = new byte[Math.min(len, 16384)];
+            int buflen = Math.min(len, BUFSIZE);
+            ByteArray ba;
+            byte[] buf;
+            if (buflen == BUFSIZE) {
+                ba = _cache.acquire();
+                buf = ba.getData();
+            } else {
+                ba = null;
+                buf = new byte[buflen];
+            }
             synchronized (this) {
                 if (raf == null)
                     throw new IOException();
@@ -220,6 +255,8 @@ class PartialPiece implements Comparable {
                     out.write(buf, 0, rd);
                 }
             }
+            if (ba != null)
+                _cache.release(ba, false);
         }
     }
     

apps/i2psnark/java/src/org/klomp/snark/Peer.java

@@ -359,7 +359,7 @@ public class Peer implements Comparable
     String bittorrentProtocol = new String(bs, "UTF-8");
     if (!"BitTorrent protocol".equals(bittorrentProtocol))
       throw new IOException("Handshake failure, expected "
-                            + "'Bittorrent protocol', got '"
+                            + "'BitTorrent protocol', got '"
                             + bittorrentProtocol + "'");
     
     // Handshake read - options

apps/i2psnark/java/src/org/klomp/snark/PeerAcceptor.java

@@ -40,7 +40,7 @@ import net.i2p.util.Log;
  * protocol connection. The PeerAcceptor will then create a new peer
  * if the PeerCoordinator wants more peers.
  */
-public class PeerAcceptor
+class PeerAcceptor
 {
   private final Log _log = I2PAppContext.getGlobalContext().logManager().getLog(PeerAcceptor.class);
   private final PeerCoordinator coordinator;

apps/i2psnark/java/src/org/klomp/snark/PeerConnectionOut.java

@@ -437,6 +437,7 @@ class PeerConnectionOut implements Runnable
    */
   void sendPiece(int piece, int begin, int length, DataLoader loader)
   {
+    /****
       boolean sendNow = false;
       // are there any cases where we should?
 
@@ -447,6 +448,7 @@ class PeerConnectionOut implements Runnable
             sendPiece(piece, begin, length, bytes);
         return;
       }
+    ****/
 
       // queue a fake message... set everything up,
       // except save the PeerState instead of the bytes.

apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java

@@ -35,6 +35,7 @@ import java.util.Set;
 import java.util.concurrent.LinkedBlockingQueue;
 
 import net.i2p.I2PAppContext;
+import net.i2p.data.ByteArray;
 import net.i2p.data.DataHelper;
 import net.i2p.data.Destination;
 import net.i2p.util.ConcurrentHashSet;
@@ -214,7 +215,7 @@ class PeerCoordinator implements PeerListener
 
   public Storage getStorage() { return storage; }
 
-  // for web page detailed stats
+  /** for web page detailed stats */
   public List<Peer> peerList()
   {
         return new ArrayList(peers);
@@ -375,8 +376,11 @@ class PeerCoordinator implements PeerListener
    */
   public boolean needOutboundPeers() {
         //return wantedBytes != 0 && needPeers();
-        // minus one to make it a little easier for new peers to get in on large swarms
-        return wantedBytes != 0 && !halted && peers.size() < getMaxConnections() - 1;
+        // minus two to make it a little easier for new peers to get in on large swarms
+        return wantedBytes != 0 &&
+               !halted &&
+               peers.size() < getMaxConnections() - 2 &&
+               (storage == null || !storage.isChecking());
   }
   
   /**
@@ -445,6 +449,12 @@ class PeerCoordinator implements PeerListener
     synchronized (downloaded_old) {
         Arrays.fill(downloaded_old, 0);
     }
+    // failsafe
+    synchronized(wantedPieces) {
+        for (Piece pc : wantedPieces) {
+            pc.clear();
+        }
+    }
     timer.schedule((CHECK_PERIOD / 2) + _random.nextInt((int) CHECK_PERIOD));
   }
 
@@ -508,8 +518,8 @@ class PeerCoordinator implements PeerListener
             peerCount = peers.size();
             unchokePeer();
 
-            if (listener != null)
-              listener.peerChange(this, peer);
+            //if (listener != null)
+            //  listener.peerChange(this, peer);
           }
       }
     if (toDisconnect != null) {
@@ -645,13 +655,18 @@ class PeerCoordinator implements PeerListener
    */
   public boolean gotHave(Peer peer, int piece)
   {
-    if (listener != null)
-      listener.peerChange(this, peer);
+    //if (listener != null)
+    //  listener.peerChange(this, peer);
 
-    synchronized(wantedPieces)
-      {
-        return wantedPieces.contains(new Piece(piece));
-      }
+    synchronized(wantedPieces) {
+        for (Piece pc : wantedPieces) {
+            if (pc.getId() == piece) {
+                pc.addPeer(peer);
+                return true;
+            }
+        }
+        return false;
+    }
   }
 
   /**
@@ -660,23 +675,20 @@ class PeerCoordinator implements PeerListener
    */
   public boolean gotBitField(Peer peer, BitField bitfield)
   {
-    if (listener != null)
-      listener.peerChange(this, peer);
+    //if (listener != null)
+    //  listener.peerChange(this, peer);
 
-    synchronized(wantedPieces)
-      {
-        Iterator<Piece> it = wantedPieces.iterator();
-        while (it.hasNext())
-          {
-            Piece p = it.next();
+    boolean rv = false;
+    synchronized(wantedPieces) {
+        for (Piece p : wantedPieces) {
             int i = p.getId();
             if (bitfield.get(i)) {
               p.addPeer(peer);
-              return true;
+              rv = true;
             }
-          }
-      }
-    return false;
+        }
+    }
+    return rv;
   }
 
   /**
@@ -732,7 +744,19 @@ class PeerCoordinator implements PeerListener
                 break;
             if (havePieces.get(p.getId()) && !p.isRequested())
               {
-                piece = p;
+                // never ever choose one that's in partialPieces, or we
+                // will create a second one and leak
+                boolean hasPartial = false;
+                for (PartialPiece pp : partialPieces) {
+                    if (pp.getPiece() == p.getId()) {
+                        if (_log.shouldLog(Log.INFO))
+                            _log.info("wantPiece() skipping partial for " + peer + ": piece = " + pp);
+                        hasPartial = true;
+                        break;
+                    }
+                }
+                if (!hasPartial)
+                    piece = p;
               }
             else if (p.isRequested()) 
             {
@@ -747,8 +771,12 @@ class PeerCoordinator implements PeerListener
             // AND if there are almost no wanted pieces left (real end game).
             // If we do end game all the time, we generate lots of extra traffic
             // when the seeder is super-slow and all the peers are "caught up"
-            if (wantedSize > END_GAME_THRESHOLD)
+            if (wantedSize > END_GAME_THRESHOLD) {
+                if (_log.shouldLog(Log.INFO))
+                    _log.info("Nothing to request, " + requested.size() + " being requested and " +
+                              wantedSize + " still wanted");
                 return null;  // nothing to request and not in end game
+            }
             // let's not all get on the same piece
             // Even better would be to sort by number of requests
             if (record)
@@ -784,7 +812,8 @@ class PeerCoordinator implements PeerListener
         }
         if (record) {
             if (_log.shouldLog(Log.INFO))
-                _log.info(peer + " is now requesting: piece " + piece + " priority " + piece.getPriority());
+                _log.info("Now requesting from " + peer + ": piece " + piece + " priority " + piece.getPriority() +
+                          " peers " + piece.getPeerCount() + '/' + peers.size());
             piece.setRequested(peer, true);
         }
         return piece;
@@ -874,7 +903,7 @@ class PeerCoordinator implements PeerListener
    *
    * @throws RuntimeException on IOE getting the data
    */
-  public byte[] gotRequest(Peer peer, int piece, int off, int len)
+  public ByteArray gotRequest(Peer peer, int piece, int off, int len)
   {
     if (halted)
       return null;
@@ -905,8 +934,8 @@ class PeerCoordinator implements PeerListener
   {
     uploaded += size;
 
-    if (listener != null)
-      listener.peerChange(this, peer);
+    //if (listener != null)
+    //  listener.peerChange(this, peer);
   }
 
   /**
@@ -916,8 +945,8 @@ class PeerCoordinator implements PeerListener
   {
     downloaded += size;
 
-    if (listener != null)
-      listener.peerChange(this, peer);
+    //if (listener != null)
+    //  listener.peerChange(this, peer);
   }
 
   /**
@@ -929,13 +958,11 @@ class PeerCoordinator implements PeerListener
    */
   public boolean gotPiece(Peer peer, PartialPiece pp)
   {
-    if (metainfo == null || storage == null)
+    if (metainfo == null || storage == null || storage.isChecking() || halted) {
+        pp.release();
         return true;
-    int piece = pp.getPiece();
-    if (halted) {
-      _log.info("Got while-halted piece " + piece + "/" + metainfo.getPieces() +" from " + peer + " for " + metainfo.getName());
-      return true; // We don't actually care anymore.
     }
+    int piece = pp.getPiece();
     
     synchronized(wantedPieces)
       {
@@ -948,12 +975,15 @@ class PeerCoordinator implements PeerListener
             // Assume we got a good piece, we don't really care anymore.
             // Well, this could be caused by a change in priorities, so
             // only return true if we already have it, otherwise might as well keep it.
-            if (storage.getBitField().get(piece))
+            if (storage.getBitField().get(piece)) {
+                pp.release();
                 return true;
+            }
           }
         
         try
           {
+            // this takes forever if complete, as it rechecks
             if (storage.putPiece(pp))
               {
                 if (_log.shouldLog(Log.INFO))
@@ -1026,8 +1056,8 @@ class PeerCoordinator implements PeerListener
     if (_log.shouldLog(Log.INFO))
       _log.info("Got choke(" + choke + "): " + peer);
 
-    if (listener != null)
-      listener.peerChange(this, peer);
+    //if (listener != null)
+    //  listener.peerChange(this, peer);
   }
 
   public void gotInterest(Peer peer, boolean interest)
@@ -1046,8 +1076,8 @@ class PeerCoordinator implements PeerListener
               }
       }
 
-    if (listener != null)
-      listener.peerChange(this, peer);
+    //if (listener != null)
+    //  listener.peerChange(this, peer);
   }
 
   public void disconnected(Peer peer)
@@ -1067,18 +1097,18 @@ class PeerCoordinator implements PeerListener
         peerCount = peers.size();
       }
 
-    if (listener != null)
-      listener.peerChange(this, peer);
+    //if (listener != null)
+    //  listener.peerChange(this, peer);
   }
   
   /** Called when a peer is removed, to prevent it from being used in 
    * rarest-first calculations.
    */
-  public void removePeerFromPieces(Peer peer) {
+  private void removePeerFromPieces(Peer peer) {
       synchronized(wantedPieces) {
-          for(Iterator<Piece> iter = wantedPieces.iterator(); iter.hasNext(); ) {
-              Piece piece = iter.next();
+          for (Piece piece : wantedPieces) {
               piece.removePeer(peer);
+              piece.setRequested(peer, false);
           }
       } 
   }
@@ -1159,6 +1189,8 @@ class PeerCoordinator implements PeerListener
   public PartialPiece getPartialPiece(Peer peer, BitField havePieces) {
       if (metainfo == null)
           return null;
+      if (storage != null && storage.isChecking())
+          return null;
       synchronized(wantedPieces) {
           // sorts by remaining bytes, least first
           Collections.sort(partialPieces);
@@ -1171,11 +1203,24 @@ class PeerCoordinator implements PeerListener
                  for(Piece piece : wantedPieces) {
                      if (piece.getId() == savedPiece) {
                          if (peer.isCompleted() && piece.getPeerCount() > 1) {
-                             // Try to preserve rarest-first when we have only one seeder
-                             // by not preferring a partial piece that others have too
+                             // Try to preserve rarest-first
+                             // by not requesting a partial piece that non-seeders also have
                              // from a seeder
-                             skipped = true;
-                             break;
+                             boolean nonSeeds = false;
+                             for (Peer pr : peers) {
+                                 PeerState state = pr.state;
+                                 if (state == null) continue;
+                                 BitField bf = state.bitfield;
+                                 if (bf == null) continue;
+                                 if (bf.get(savedPiece) && !pr.isCompleted()) {
+                                     nonSeeds = true;
+                                     break;
+                                 }
+                             }
+                             if (nonSeeds) {
+                                 skipped = true;
+                                 break;
+                             }
                          }
                          iter.remove();
                          piece.setRequested(peer, true);
@@ -1250,6 +1295,7 @@ class PeerCoordinator implements PeerListener
               PartialPiece pp = iter.next();
               if (pp.getPiece() == piece) {
                   iter.remove();
+                  pp.release();
                   // there should be only one but keep going to be sure
               }
           }
@@ -1393,6 +1439,7 @@ class PeerCoordinator implements PeerListener
 
   /**
    *  Called by TrackerClient
+   *  @return the Set itself, modifiable, not a copy, caller should clear()
    *  @since 0.8.4
    */
   Set<PeerID> getPEXPeers() {

apps/i2psnark/java/src/org/klomp/snark/PeerCoordinatorSet.java

@@ -12,7 +12,7 @@ import net.i2p.crypto.SHA1Hash;
  * Each PeerCoordinator is added to the set from within the Snark (and removed
  * from it there too)
  */
-public class PeerCoordinatorSet {
+class PeerCoordinatorSet {
     private final Map<SHA1Hash, PeerCoordinator> _coordinators;
     
     public PeerCoordinatorSet() {

apps/i2psnark/java/src/org/klomp/snark/PeerID.java

@@ -43,7 +43,7 @@ import org.klomp.snark.bencode.InvalidBEncodingException;
  *  and the PeerID is not required.
  *  Equality is now determined solely by the dest hash.
  */
-public class PeerID implements Comparable
+class PeerID implements Comparable
 {
   private byte[] id;
   private Destination address;

apps/i2psnark/java/src/org/klomp/snark/PeerListener.java

@@ -22,6 +22,8 @@ package org.klomp.snark;
 
 import java.util.List;
 
+import net.i2p.data.ByteArray;
+
 /**
  * Listener for Peer events.
  */
@@ -114,7 +116,7 @@ interface PeerListener
    * @return a byte array containing the piece or null when the piece
    * is not available (which is a protocol error).
    */
-  byte[] gotRequest(Peer peer, int piece, int off, int len);
+  ByteArray gotRequest(Peer peer, int piece, int off, int len);
 
   /**
    * Called when a (partial) piece has been downloaded from the peer.

apps/i2psnark/java/src/org/klomp/snark/PeerState.java

@@ -27,6 +27,7 @@ import java.util.List;
 import java.util.Set;
 
 import net.i2p.I2PAppContext;
+import net.i2p.data.ByteArray;
 import net.i2p.util.Log;
 
 class PeerState implements DataLoader
@@ -245,8 +246,8 @@ class PeerState implements DataLoader
    *  @return bytes or null for errors
    *  @since 0.8.2
    */
-  public byte[] loadData(int piece, int begin, int length) {
-    byte[] pieceBytes = listener.gotRequest(peer, piece, begin, length);
+  public ByteArray loadData(int piece, int begin, int length) {
+    ByteArray pieceBytes = listener.gotRequest(peer, piece, begin, length);
     if (pieceBytes == null)
       {
         // XXX - Protocol error-> diconnect?
@@ -256,7 +257,7 @@ class PeerState implements DataLoader
       }
 
     // More sanity checks
-    if (length != pieceBytes.length)
+    if (length != pieceBytes.getData().length)
       {
         // XXX - Protocol error-> disconnect?
         if (_log.shouldLog(Log.WARN))
@@ -355,22 +356,21 @@ class PeerState implements DataLoader
                   + piece + "," + begin + "," + length + ") from "
                   + peer);
 
-    int r = getFirstOutstandingRequest(piece);
-
-    // Unrequested piece number?
-    if (r == -1)
-      {
-        if (_log.shouldLog(Log.INFO))
-          _log.info("Unrequested 'piece: " + piece + ", "
-                      + begin + ", " + length + "' received from "
-                      + peer);
-        return null;
-      }
-
     // Lookup the correct piece chunk request from the list.
     Request req;
     synchronized(this)
       {
+        int r = getFirstOutstandingRequest(piece);
+
+        // Unrequested piece number?
+        if (r == -1) {
+            if (_log.shouldLog(Log.INFO))
+                _log.info("Unrequested 'piece: " + piece + ", "
+                      + begin + ", " + length + "' received from "
+                      + peer);
+            return null;
+        }
+
         req = outstandingRequests.get(r);
         while (req.getPiece() == piece && req.off != begin
                && r < outstandingRequests.size() - 1)
@@ -591,6 +591,7 @@ class PeerState implements DataLoader
                 // Send cancel even when we are choked to make sure that it is
                 // really never ever send.
                 out.sendCancel(req);
+                req.getPartialPiece().release();
               }
           }
   }
@@ -681,6 +682,7 @@ class PeerState implements DataLoader
                   _log.debug(peer + " addRequest() we are choked, delaying requestNextPiece()");
               return;
           }
+          // huh? rv unused
           more_pieces = requestNextPiece();
         } else if (more_pieces) // We want something
           {
@@ -710,6 +712,8 @@ class PeerState implements DataLoader
       }
 
     // failsafe
+    // However this is bad as it thrashes the peer when we change our mind
+    // Ticket 691 cause here?
     if (interesting && lastRequest == null && outstandingRequests.isEmpty())
         setInteresting(false);
 
@@ -737,6 +741,10 @@ class PeerState implements DataLoader
                   out.sendRequest(r);
                 lastRequest = r;
                 return true;
+            } else {
+                if (_log.shouldLog(Log.WARN))
+                    _log.warn("Got dup from coord: " + pp);
+                pp.release();
             }
         }
 
@@ -783,6 +791,8 @@ class PeerState implements DataLoader
     }
 
     // failsafe
+    // However this is bad as it thrashes the peer when we change our mind
+    // Ticket 691 cause here?
     if (outstandingRequests.isEmpty())
         lastRequest = null;
 

apps/i2psnark/java/src/org/klomp/snark/Piece.java

@@ -12,13 +12,13 @@ class Piece implements Comparable {
     private final int id;
     private final Set<PeerID> peers;
     /** @since 0.8.3 */
-    private Set<PeerID> requests;
+    private volatile Set<PeerID> requests;
     /** @since 0.8.1 */
     private int priority;
     
     public Piece(int id) {
         this.id = id;
-        this.peers = new HashSet(I2PSnarkUtil.MAX_CONNECTIONS);
+        this.peers = new HashSet(I2PSnarkUtil.MAX_CONNECTIONS / 2);
         // defer creating requests to save memory
     }
     
@@ -54,7 +54,10 @@ class Piece implements Comparable {
     /** caller must synchronize */
     public boolean addPeer(Peer peer) { return this.peers.add(peer.getPeerID()); }
 
-    /** caller must synchronize */
+    /**
+     * Caller must synchronize.
+     * @return true if removed
+     */
     public boolean removePeer(Peer peer) { return this.peers.remove(peer.getPeerID()); }
 
     /**
@@ -104,6 +107,17 @@ class Piece implements Comparable {
     public int getRequestCount() {
         return this.requests == null ? 0 : this.requests.size();
     } 
+
+    /**
+     * Clear all knowledge of peers
+     * Caller must synchronize
+     * @since 0.9.3
+     */
+    public void clear() {
+        peers.clear();
+        if (requests != null)
+            requests.clear();
+    } 
     
     /** @return default 0 @since 0.8.1 */
     public int getPriority() { return this.priority; }

apps/i2psnark/java/src/org/klomp/snark/Snark.java

@@ -688,6 +688,22 @@ public class Snark
         starting = true;
     }
 
+    /**
+     *  File checking in progress.
+     *  @since 0.9.3
+     */
+    public boolean isChecking() {
+        return storage != null && storage.isChecking();
+    }
+
+    /**
+     *  Disk allocation (ballooning) in progress.
+     *  @since 0.9.3
+     */
+    public boolean isAllocating() {
+        return storage != null && storage.isAllocating();
+    }
+
     /**
      *  @since 0.8.4
      */
@@ -1099,7 +1115,12 @@ public class Snark
       }
   }
 
+
+  ///////////// Begin StorageListener methods
+
   //private boolean allocating = false;
+
+  /** does nothing */
   public void storageCreateFile(Storage storage, String name, long length)
   {
     //if (allocating)
@@ -1113,6 +1134,7 @@ public class Snark
   // How much storage space has been allocated
   private long allocated = 0;
 
+  /** does nothing */
   public void storageAllocated(Storage storage, long length)
   {
     //allocating = true;
@@ -1124,7 +1146,8 @@ public class Snark
 
   private boolean allChecked = false;
   private boolean checking = false;
-  private boolean prechecking = true;
+  //private boolean prechecking = true;
+
   public void storageChecked(Storage storage, int num, boolean checked)
   {
     //allocating = false;
@@ -1142,6 +1165,8 @@ public class Snark
     if (!checking) {
         if (_log.shouldLog(Log.INFO))
             _log.info("Got " + (checked ? "" : "BAD ") + "piece: " + num);
+        if (completeListener != null)
+            completeListener.gotPiece(this);
     }
   }
 
@@ -1171,6 +1196,9 @@ public class Snark
     coordinator.setWantedPieces();
   }
 
+  ///////////// End StorageListener methods
+
+
   /** SnarkSnutdown callback unused */
   public void shutdown()
   {
@@ -1188,34 +1216,6 @@ public class Snark
           completeListener.addMessage(this, message);
   }
 
-  public interface CompleteListener {
-    public void torrentComplete(Snark snark);
-    public void updateStatus(Snark snark);
-
-    /**
-     * We transitioned from magnet mode, we have now initialized our
-     * metainfo and storage. The listener should now call getMetaInfo()
-     * and save the data to disk.
-     *
-     * @return the new name for the torrent or null on error
-     * @since 0.8.4
-     */
-    public String gotMetaInfo(Snark snark);
-
-    /**
-     * @since 0.9
-     */
-    public void fatal(Snark snark, String error);
-
-    /**
-     * @since 0.9.2
-     */
-    public void addMessage(Snark snark, String message);
-
-    // not really listeners but the easiest way to get back to an optional SnarkManager
-    public long getSavedTorrentTime(Snark snark);
-    public BitField getSavedTorrentBitField(Snark snark);
-  }
 
   /** Maintain a configurable total uploader cap
    * coordinatorListener

apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java

@@ -27,6 +27,7 @@ import java.util.concurrent.LinkedBlockingQueue;
 import net.i2p.I2PAppContext;
 import net.i2p.data.Base64;
 import net.i2p.data.DataHelper;
+import net.i2p.update.*;
 import net.i2p.util.ConcurrentHashSet;
 import net.i2p.util.FileUtil;
 import net.i2p.util.I2PAppThread;
@@ -42,7 +43,7 @@ import org.klomp.snark.dht.DHT;
 /**
  * Manage multiple snarks
  */
-public class SnarkManager implements Snark.CompleteListener {
+public class SnarkManager implements CompleteListener {
     
     /**
      *  Map of (canonical) filename of the .torrent file to Snark instance.
@@ -56,6 +57,8 @@ public class SnarkManager implements Snark.CompleteListener {
     private /* FIXME final FIXME */ File _configFile;
     private Properties _config;
     private final I2PAppContext _context;
+    private final String _contextPath;
+    private final String _contextName;
     private final Log _log;
     private final Queue<String> _messages;
     private final I2PSnarkUtil _util;
@@ -65,6 +68,8 @@ public class SnarkManager implements Snark.CompleteListener {
     private volatile boolean _running;
     private volatile boolean _stopping;
     private final Map<String, Tracker> _trackerMap;
+    private UpdateManager _umgr;
+    private UpdateHandler _uhandler;
     
     public static final String PROP_I2CP_HOST = "i2psnark.i2cpHost";
     public static final String PROP_I2CP_PORT = "i2psnark.i2cpPort";
@@ -79,7 +84,7 @@ public class SnarkManager implements Snark.CompleteListener {
     public static final String PROP_META_PRIORITY_SUFFIX = ".priority";
     public static final String PROP_META_MAGNET_PREFIX = "i2psnark.magnet.";
 
-    private static final String CONFIG_FILE = "i2psnark.config";
+    private static final String CONFIG_FILE_SUFFIX = ".config";
     public static final String PROP_FILES_PUBLIC = "i2psnark.filesPublic";
     public static final String PROP_AUTO_START = "i2snark.autoStart";   // oops
     public static final String DEFAULT_AUTO_START = "false";
@@ -87,6 +92,7 @@ public class SnarkManager implements Snark.CompleteListener {
     //public static final String DEFAULT_LINK_PREFIX = "file:///";
     public static final String PROP_STARTUP_DELAY = "i2psnark.startupDelay";
     public static final String PROP_REFRESH_DELAY = "i2psnark.refreshSeconds";
+    public static final String PROP_PAGE_SIZE = "i2psnark.pageSize";
     public static final String RC_PROP_THEME = "routerconsole.theme";
     public static final String RC_PROP_UNIVERSAL_THEMING = "routerconsole.universal.theme";
     public static final String PROP_THEME = "i2psnark.theme";
@@ -100,6 +106,7 @@ public class SnarkManager implements Snark.CompleteListener {
     public static final int DEFAULT_MAX_UP_BW = 10;
     public static final int DEFAULT_STARTUP_DELAY = 3; 
     public static final int DEFAULT_REFRESH_DELAY_SECS = 60;
+    private static final int DEFAULT_PAGE_SIZE = 50;
 
     /**
      *  "name", "announceURL=websiteURL" pairs
@@ -117,7 +124,7 @@ public class SnarkManager implements Snark.CompleteListener {
 //       , "Galen", "http://5jpwQMI5FT303YwKa5Rd38PYSX04pbIKgTaKQsWbqoWjIfoancFdWCShXHLI5G5ofOb0Xu11vl2VEMyPsg1jUFYSVnu4-VfMe3y4TKTR6DTpetWrnmEK6m2UXh91J5DZJAKlgmO7UdsFlBkQfR2rY853-DfbJtQIFl91tbsmjcA5CGQi4VxMFyIkBzv-pCsuLQiZqOwWasTlnzey8GcDAPG1LDcvfflGV~6F5no9mnuisZPteZKlrv~~TDoXTj74QjByWc4EOYlwqK8sbU9aOvz~s31XzErbPTfwiawiaZ0RUI-IDrKgyvmj0neuFTWgjRGVTH8bz7cBZIc3viy6ioD-eMQOrXaQL0TCWZUelRwHRvgdPiQrxdYQs7ixkajeHzxi-Pq0EMm5Vbh3j3Q9kfUFW3JjFDA-MLB4g6XnjCbM5J1rC0oOBDCIEfhQkszru5cyLjHiZ5yeA0VThgu~c7xKHybv~OMXION7V8pBKOgET7ZgAkw1xgYe3Kkyq5syAAAA.i2p/tr/announce.php=http://galen.i2p/tr/"
        "Postman", "http://tracker2.postman.i2p/announce.php=http://tracker2.postman.i2p/"
        ,"Welterde", "http://tracker.welterde.i2p/a=http://tracker.welterde.i2p/stats?mode=top5"
-       ,"Diftracker", "http://n--XWjHjUPjnMNrSwXA2OYXpMIUL~u4FNXnrt2HtjK3y6j~4SOClyyeKzd0zRPlixxkCe2wfBIYye3bZsaqAD8bd0QMmowxbq91WpjsPfKMiphJbePKXtYAVARiy0cqyvh1d2LyDE-6wkvgaw45hknmS0U-Dg3YTJZbAQRU2SKXgIlAbWCv4R0kDFBLEVpReDiJef3rzAWHiW8yjmJuJilkYjMwlfRjw8xx1nl2s~yhlljk1pl13jGYb0nfawQnuOWeP-ASQWvAAyVgKvZRJE2O43S7iveu9piuv7plXWbt36ef7ndu2GNoNyPOBdpo9KUZ-NOXm4Kgh659YtEibL15dEPAOdxprY0sYUurVw8OIWqrpX7yn08nbi6qHVGqQwTpxH35vkL8qrCbm-ym7oQJQnNmSDrNTyWYRFSq5s5~7DAdFDzqRPW-pX~g0zEivWj5tzkhvG9rVFgFo0bpQX3X0PUAV9Xbyf8u~v8Zbr9K1pCPqBq9XEr4TqaLHw~bfAAAA.i2p/announce.php=http://diftracker.i2p/"
+       ,"Diftracker", "http://diftracker.i2p/announce.php=http://diftracker.i2p/"
 //       , "CRSTRACK", "http://b4G9sCdtfvccMAXh~SaZrPqVQNyGQbhbYMbw6supq2XGzbjU4NcOmjFI0vxQ8w1L05twmkOvg5QERcX6Mi8NQrWnR0stLExu2LucUXg1aYjnggxIR8TIOGygZVIMV3STKH4UQXD--wz0BUrqaLxPhrm2Eh9Hwc8TdB6Na4ShQUq5Xm8D4elzNUVdpM~RtChEyJWuQvoGAHY3ppX-EJJLkiSr1t77neS4Lc-KofMVmgI9a2tSSpNAagBiNI6Ak9L1T0F9uxeDfEG9bBSQPNMOSUbAoEcNxtt7xOW~cNOAyMyGydwPMnrQ5kIYPY8Pd3XudEko970vE0D6gO19yoBMJpKx6Dh50DGgybLQ9CpRaynh2zPULTHxm8rneOGRcQo8D3mE7FQ92m54~SvfjXjD2TwAVGI~ae~n9HDxt8uxOecAAvjjJ3TD4XM63Q9TmB38RmGNzNLDBQMEmJFpqQU8YeuhnS54IVdUoVQFqui5SfDeLXlSkh4vYoMU66pvBfWbAAAA.i2p/tracker/announce.php=http://crstrack.i2p/tracker/"
 //       ,"Exotrack", "http://blbgywsjubw3d2zih2giokakhe3o2cko7jtte4risb3hohbcoyva.b32.i2p/announce.php=http://exotrack.i2p/"
     };
@@ -125,17 +132,33 @@ public class SnarkManager implements Snark.CompleteListener {
     /** comma delimited list of name=announceURL=baseURL for the trackers to be displayed */
     public static final String PROP_TRACKERS = "i2psnark.trackers";
 
+    /**
+     *  For embedded.
+     */
     public SnarkManager(I2PAppContext ctx) {
+        this(ctx, "/i2psnark", "i2psnark");
+    }
+
+    /**
+     *  For webapp.
+     *  @param ctxPath generally "/i2psnark"
+     *  @param ctxName generally "i2psnark"
+     *  @since 0.9.6
+     */
+    public SnarkManager(I2PAppContext ctx, String ctxPath, String ctxName) {
         _snarks = new ConcurrentHashMap();
         _magnets = new ConcurrentHashSet();
         _addSnarkLock = new Object();
         _context = ctx;
+        _contextPath = ctxPath;
+        _contextName = ctxName;
         _log = _context.logManager().getLog(SnarkManager.class);
         _messages = new LinkedBlockingQueue();
-        _util = new I2PSnarkUtil(_context);
-        _configFile = new File(CONFIG_FILE);
+        _util = new I2PSnarkUtil(_context, ctxName);
+        String cfile = ctxName + CONFIG_FILE_SUFFIX;
+        _configFile = new File(cfile);
         if (!_configFile.isAbsolute())
-            _configFile = new File(_context.getConfigDir(), CONFIG_FILE);
+            _configFile = new File(_context.getConfigDir(), cfile);
         _trackerMap = new ConcurrentHashMap(4);
         loadConfig(null);
     }
@@ -149,10 +172,30 @@ public class SnarkManager implements Snark.CompleteListener {
         _connectionAcceptor = new ConnectionAcceptor(_util);
         _monitor = new I2PAppThread(new DirMonitor(), "Snark DirMonitor", true);
         _monitor.start();
+        // only if default instance
+        if ("i2psnark".equals(_contextName))
+            // delay until UpdateManager is there
+            _context.simpleScheduler().addEvent(new Register(), 4*60*1000);
         // Not required, Jetty has a shutdown hook
         //_context.addShutdownTask(new SnarkManagerShutdown());
     }
 
+    /** @since 0.9.4 */
+    private class Register implements SimpleTimer.TimedEvent {
+        public void timeReached() {
+            if (!_running)
+                return;
+            _umgr = _context.updateManager();
+            if (_umgr != null) {
+                _uhandler = new UpdateHandler(_context, _umgr, SnarkManager.this);
+                _umgr.register(_uhandler, UpdateType.ROUTER_SIGNED, UpdateMethod.TORRENT, 10);
+                _log.warn("Registering with update manager");
+            } else {
+                _log.warn("No update manager to register with");
+            }
+        }
+    }
+    
     /*
      *  Called by the webapp at Jetty shutdown.
      *  Stops all torrents. Does not close the tunnel, so the announces have a chance.
@@ -160,6 +203,10 @@ public class SnarkManager implements Snark.CompleteListener {
      *  Runs inline.
      */
     public void stop() {
+        if (_umgr != null && _uhandler != null) {
+            //_uhandler.shutdown();
+            _umgr.unregister(_uhandler, UpdateType.ROUTER_SIGNED, UpdateMethod.TORRENT);
+        }
         _running = false;
         _monitor.interrupt();
         _connectionAcceptor.halt();
@@ -200,11 +247,11 @@ public class SnarkManager implements Snark.CompleteListener {
      *  @since 0.8.9
      */
     public boolean areFilesPublic() {
-        return Boolean.valueOf(_config.getProperty(PROP_FILES_PUBLIC)).booleanValue();
+        return Boolean.parseBoolean(_config.getProperty(PROP_FILES_PUBLIC));
     }
 
     public boolean shouldAutoStart() {
-        return Boolean.valueOf(_config.getProperty(PROP_AUTO_START, DEFAULT_AUTO_START)).booleanValue();
+        return Boolean.parseBoolean(_config.getProperty(PROP_AUTO_START, DEFAULT_AUTO_START));
     }
 
 /****
@@ -225,6 +272,18 @@ public class SnarkManager implements Snark.CompleteListener {
         }
     }
 
+    /**
+     *  For GUI
+     *  @since 0.9.6
+     */
+    public int getPageSize() { 
+        try {
+            return Integer.parseInt(_config.getProperty(PROP_PAGE_SIZE));
+        } catch (NumberFormatException nfe) {
+            return DEFAULT_PAGE_SIZE;
+        }
+    }
+
     private int getStartupDelayMinutes() { 
         try {
 	    return Integer.parseInt(_config.getProperty(PROP_STARTUP_DELAY));
@@ -234,7 +293,7 @@ public class SnarkManager implements Snark.CompleteListener {
     }
 
     public File getDataDir() { 
-        String dir = _config.getProperty(PROP_DIR, "i2psnark");
+        String dir = _config.getProperty(PROP_DIR, _contextName);
         File f;
         if (areFilesPublic())
             f = new File(dir);
@@ -280,13 +339,15 @@ public class SnarkManager implements Snark.CompleteListener {
         if (!_config.containsKey(PROP_UPLOADERS_TOTAL))
             _config.setProperty(PROP_UPLOADERS_TOTAL, "" + Snark.MAX_TOTAL_UPLOADERS);
         if (!_config.containsKey(PROP_DIR))
-            _config.setProperty(PROP_DIR, "i2psnark");
+            _config.setProperty(PROP_DIR, _contextName);
         if (!_config.containsKey(PROP_AUTO_START))
             _config.setProperty(PROP_AUTO_START, DEFAULT_AUTO_START);
         if (!_config.containsKey(PROP_REFRESH_DELAY))
             _config.setProperty(PROP_REFRESH_DELAY, Integer.toString(DEFAULT_REFRESH_DELAY_SECS));
         if (!_config.containsKey(PROP_STARTUP_DELAY))
             _config.setProperty(PROP_STARTUP_DELAY, Integer.toString(DEFAULT_STARTUP_DELAY));
+        if (!_config.containsKey(PROP_PAGE_SIZE))
+            _config.setProperty(PROP_PAGE_SIZE, Integer.toString(DEFAULT_PAGE_SIZE));
         if (!_config.containsKey(PROP_THEME))
             _config.setProperty(PROP_THEME, DEFAULT_THEME);
         // no, so we can switch default to true later
@@ -384,11 +445,11 @@ public class SnarkManager implements Snark.CompleteListener {
         if (ot != null)
             _util.setOpenTrackers(getOpenTrackers());
         String useOT = _config.getProperty(PROP_USE_OPENTRACKERS);
-        boolean bOT = useOT == null || Boolean.valueOf(useOT).booleanValue();
+        boolean bOT = useOT == null || Boolean.parseBoolean(useOT);
         _util.setUseOpenTrackers(bOT);
         // careful, so we can switch default to true later
-        _util.setUseDHT(Boolean.valueOf(_config.getProperty(PROP_USE_DHT,
-                                          Boolean.toString(I2PSnarkUtil.DEFAULT_USE_DHT))).booleanValue());
+        _util.setUseDHT(Boolean.parseBoolean(_config.getProperty(PROP_USE_DHT,
+                                          Boolean.toString(I2PSnarkUtil.DEFAULT_USE_DHT))));
         getDataDir().mkdirs();
         initTrackerMap();
     }
@@ -404,11 +465,15 @@ public class SnarkManager implements Snark.CompleteListener {
         return defaultVal;
     }
     
+    /**
+     *  all params may be null or need trimming
+     */
     public void updateConfig(String dataDir, boolean filesPublic, boolean autoStart, String refreshDelay,
-                             String startDelay, String seedPct, String eepHost, 
+                             String startDelay, String pageSize, String seedPct, String eepHost, 
                              String eepPort, String i2cpHost, String i2cpPort, String i2cpOpts,
                              String upLimit, String upBW, boolean useOpenTrackers, boolean useDHT, String theme) {
         boolean changed = false;
+        boolean interruptMonitor = false;
         //if (eepHost != null) {
         //    // unused, we use socket eepget
         //    int port = _util.getEepProxyPort();
@@ -425,12 +490,12 @@ public class SnarkManager implements Snark.CompleteListener {
         //}
         if (upLimit != null) {
             int limit = _util.getMaxUploaders();
-            try { limit = Integer.parseInt(upLimit); } catch (NumberFormatException nfe) {}
+            try { limit = Integer.parseInt(upLimit.trim()); } catch (NumberFormatException nfe) {}
             if ( limit != _util.getMaxUploaders()) {
                 if ( limit >= Snark.MIN_TOTAL_UPLOADERS ) {
                     _util.setMaxUploaders(limit);
                     changed = true;
-                    _config.setProperty(PROP_UPLOADERS_TOTAL, "" + limit);
+                    _config.setProperty(PROP_UPLOADERS_TOTAL, Integer.toString(limit));
                     addMessage(_("Total uploaders limit changed to {0}", limit));
                 } else {
                     addMessage(_("Minimum total uploaders limit is {0}", Snark.MIN_TOTAL_UPLOADERS));
@@ -439,12 +504,12 @@ public class SnarkManager implements Snark.CompleteListener {
         }
         if (upBW != null) {
             int limit = _util.getMaxUpBW();
-            try { limit = Integer.parseInt(upBW); } catch (NumberFormatException nfe) {}
+            try { limit = Integer.parseInt(upBW.trim()); } catch (NumberFormatException nfe) {}
             if ( limit != _util.getMaxUpBW()) {
                 if ( limit >= MIN_UP_BW ) {
                     _util.setMaxUpBW(limit);
                     changed = true;
-                    _config.setProperty(PROP_UPBW_MAX, "" + limit);
+                    _config.setProperty(PROP_UPBW_MAX, Integer.toString(limit));
                     addMessage(_("Up BW limit changed to {0}KBps", limit));
                 } else {
                     addMessage(_("Minimum up bandwidth limit is {0}KBps", MIN_UP_BW));
@@ -454,21 +519,21 @@ public class SnarkManager implements Snark.CompleteListener {
         
 	if (startDelay != null){
 		int minutes = _util.getStartupDelay();
-                try { minutes = Integer.parseInt(startDelay); } catch (NumberFormatException nfe) {}
+                try { minutes = Integer.parseInt(startDelay.trim()); } catch (NumberFormatException nfe) {}
 	        if ( minutes != _util.getStartupDelay()) {
                 	    _util.setStartupDelay(minutes);
 	                    changed = true;
-        	            _config.setProperty(PROP_STARTUP_DELAY, "" + minutes);
+        	            _config.setProperty(PROP_STARTUP_DELAY, Integer.toString(minutes));
                 	    addMessage(_("Startup delay changed to {0}", DataHelper.formatDuration2(minutes * 60 * 1000)));
                 }
 	}
 
 	if (refreshDelay != null) {
 	    try {
-                int secs = Integer.parseInt(refreshDelay);
+                int secs = Integer.parseInt(refreshDelay.trim());
 	        if (secs != getRefreshDelaySeconds()) {
 	            changed = true;
-	            _config.setProperty(PROP_REFRESH_DELAY, refreshDelay);
+	            _config.setProperty(PROP_REFRESH_DELAY, Integer.toString(secs));
                     if (secs >= 0)
 	                addMessage(_("Refresh time changed to {0}", DataHelper.formatDuration2(secs * 1000)));
 	            else
@@ -477,6 +542,42 @@ public class SnarkManager implements Snark.CompleteListener {
 	    } catch (NumberFormatException nfe) {}
 	}
 
+        if (pageSize != null) {
+            try {
+                int size = Integer.parseInt(pageSize.trim());
+                if (size <= 0)
+                    size = 999999;
+                else if (size < 5)
+                    size = 5;
+                if (size != getPageSize()) {
+                    changed = true;
+                    pageSize = Integer.toString(size);
+                    _config.setProperty(PROP_PAGE_SIZE, pageSize);
+                    addMessage(_("Page size changed to {0}", pageSize));
+                }
+            } catch (NumberFormatException nfe) {}
+        }
+
+        if (dataDir != null && !dataDir.equals(getDataDir().getAbsolutePath())) {
+            dataDir = dataDir.trim();
+            File dd = new File(dataDir);
+            if (!dd.isAbsolute()) {
+                addMessage(_("Data directory must be an absolute path") + ": " + dataDir);
+            } else if (!dd.exists()) {
+                addMessage(_("Data directory does not exist") + ": " + dataDir);
+            } else if (!dd.isDirectory()) {
+                addMessage(_("Not a directory") + ": " + dataDir);
+            } else if (!dd.canRead()) {
+                addMessage(_("Unreadable") + ": " + dataDir);
+            } else {
+                changed = true;
+                interruptMonitor = true;
+                _config.setProperty(PROP_DIR, dataDir);
+                addMessage(_("Data directory changed to {0}", dataDir));
+            }
+
+        }
+
 	// Start of I2CP stuff.
 	// i2cpHost will generally be null since it is hidden from the form if in router context.
 
@@ -611,6 +712,9 @@ public class SnarkManager implements Snark.CompleteListener {
         }
         if (changed) {
             saveConfig();
+            if (interruptMonitor)
+                // Data dir changed. this will stop and remove all old torrents, and add the new ones
+                _monitor.interrupt();
         } else {
             addMessage(_("Configuration unchanged."));
         }
@@ -706,8 +810,13 @@ public class SnarkManager implements Snark.CompleteListener {
     
     public Properties getConfig() { return _config; }
     
+    /** @since Jetty 7 */
+    public String getConfigFilename() {
+        return _configFile.getAbsolutePath();
+    }
+
     /** hardcoded for sanity.  perhaps this should be customizable, for people who increase their ulimit, etc. */
-    private static final int MAX_FILES_PER_TORRENT = 512;
+    public static final int MAX_FILES_PER_TORRENT = 512;
     
     /**
      *  Set of canonical .torrent filenames that we are dealing with.
@@ -723,6 +832,14 @@ public class SnarkManager implements Snark.CompleteListener {
      */
     public Snark getTorrent(String filename) { synchronized (_snarks) { return _snarks.get(filename); } }
 
+    /**
+     *  Unmodifiable
+     *  @since 0.9.4
+     */
+    public Collection<Snark> getTorrents() {
+        return Collections.unmodifiableCollection(_snarks.values());
+    }
+
     /**
      * Grab the torrent given the base name of the storage
      * @return Snark or null
@@ -853,7 +970,9 @@ public class SnarkManager implements Snark.CompleteListener {
                         }
                     }
                 } catch (IOException ioe) {
-                    addMessage(_("Torrent in \"{0}\" is invalid", sfile.getName()) + ": " + ioe.getMessage());
+                    String err = _("Torrent in \"{0}\" is invalid", sfile.getName()) + ": " + ioe.getMessage();
+                    addMessage(err);
+                    _log.error(err, ioe);
                     if (sfile.exists())
                         sfile.delete();
                     return;
@@ -889,7 +1008,27 @@ public class SnarkManager implements Snark.CompleteListener {
      * @since 0.8.4
      */
     public void addMagnet(String name, byte[] ih, String trackerURL, boolean updateStatus) {
-        Snark torrent = new Snark(_util, name, ih, trackerURL, this,
+        addMagnet(name, ih, trackerURL, updateStatus, shouldAutoStart(), this);
+    }
+    
+    /**
+     * Add a torrent with the info hash alone (magnet / maggot)
+     * External use is for UpdateRunner.
+     *
+     * @param name hex or b32 name from the magnet link
+     * @param ih 20 byte info hash
+     * @param trackerURL may be null
+     * @param updateStatus should we add this magnet to the config file,
+     *                     to save it across restarts, in case we don't get
+     *                     the metadata before shutdown?
+     * @param listener to intercept callbacks, should pass through to this
+     * @return the new Snark or null on failure
+     * @throws RuntimeException via Snark.fatal()
+     * @since 0.9.4
+     */
+    public Snark addMagnet(String name, byte[] ih, String trackerURL, boolean updateStatus,
+                          boolean autoStart, CompleteListener listener) {
+        Snark torrent = new Snark(_util, name, ih, trackerURL, listener,
                                   _peerCoordinatorSet, _connectionAcceptor,
                                   false, getDataDir().getPath());
 
@@ -897,7 +1036,7 @@ public class SnarkManager implements Snark.CompleteListener {
             Snark snark = getTorrentByInfoHash(ih);
             if (snark != null) {
                 addMessage(_("Torrent with this info hash is already running: {0}", snark.getBaseName()));
-                return;
+                return null;
             }
             // Tell the dir monitor not to delete us
             _magnets.add(name);
@@ -905,8 +1044,8 @@ public class SnarkManager implements Snark.CompleteListener {
                 saveMagnetStatus(ih);
             _snarks.put(name, torrent);
         }
-        if (shouldAutoStart()) {
-            torrent.startTorrent();
+        if (autoStart) {
+            startTorrent(ih);
             addMessage(_("Fetching {0}", name));
             DHT dht = _util.getDHT();
             boolean shouldWarn = _util.connected() &&
@@ -918,7 +1057,8 @@ public class SnarkManager implements Snark.CompleteListener {
             }
         } else {
             addMessage(_("Adding {0}", name));
-      }
+        }
+        return torrent;
     }
 
     /**
@@ -1370,6 +1510,8 @@ public class SnarkManager implements Snark.CompleteListener {
                     } catch (Exception e) {
                         _log.error("Error in the DirectoryMonitor", e);
                     }
+                    if (!_snarks.isEmpty())
+                        addMessage(_("Up bandwidth limit is {0} KBps", _util.getMaxUpBW()));
                 }
                 try { Thread.sleep(60*1000); } catch (InterruptedException ie) {}
             }
@@ -1387,7 +1529,7 @@ public class SnarkManager implements Snark.CompleteListener {
         if (meta == null || storage == null)
             return;
         StringBuilder buf = new StringBuilder(256);
-        buf.append("<a href=\"/i2psnark/").append(storage.getBaseName());
+        buf.append("<a href=\"").append(_contextPath).append('/').append(storage.getBaseName());
         if (meta.getFiles() != null)
             buf.append('/');
         buf.append("\">").append(storage.getBaseName()).append("</a>");
@@ -1469,6 +1611,12 @@ public class SnarkManager implements Snark.CompleteListener {
         addMessage(message);
     }
 
+    /**
+     * A Snark.CompleteListener method.
+     * @since 0.9.4
+     */
+    public void gotPiece(Snark snark) {}
+
     // End Snark.CompleteListeners
 
     /**

apps/i2psnark/java/src/org/klomp/snark/Storage.java

@@ -21,6 +21,7 @@
 package org.klomp.snark;
 
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.RandomAccessFile;
 import java.nio.charset.Charset;
@@ -32,17 +33,23 @@ import java.util.List;
 import java.util.Map;
 import java.util.StringTokenizer;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
 
+import net.i2p.I2PAppContext;
 import net.i2p.crypto.SHA1;
+import net.i2p.data.ByteArray;
+import net.i2p.data.DataHelper;
+import net.i2p.util.ByteCache;
 import net.i2p.util.Log;
 import net.i2p.util.SecureFile;
+import net.i2p.util.SystemVersion;
 
 /**
  * Maintains pieces on disk. Can be used to store and retrieve pieces.
  */
 public class Storage
 {
-  private MetaInfo metainfo;
+  private final MetaInfo metainfo;
   private long[] lengths;
   private RandomAccessFile[] rafs;
   private String[] names;
@@ -66,9 +73,11 @@ public class Storage
   private final int pieces;
   private final long total_length;
   private boolean changed;
+  private volatile boolean _isChecking;
+  private final AtomicInteger _allocateCount = new AtomicInteger();
 
   /** The default piece size. */
-  private static final int MIN_PIECE_SIZE = 256*1024;
+  private static final int DEFAULT_PIECE_SIZE = 256*1024;
   /** note that we start reducing max number of peer connections above 1MB */
   public static final int MAX_PIECE_SIZE = 2*1024*1024;
   /** The maximum number of pieces in a torrent. */
@@ -77,23 +86,24 @@ public class Storage
 
   private static final Map<String, String> _filterNameCache = new ConcurrentHashMap();
 
-  private static final boolean _isWindows = System.getProperty("os.name").startsWith("Win");
+  private static final boolean _isWindows = SystemVersion.isWindows();
+
+  private static final int BUFSIZE = PeerState.PARTSIZE;
+  private static final ByteCache _cache = ByteCache.getInstance(16, BUFSIZE);
 
   /**
    * Creates a new storage based on the supplied MetaInfo.  This will
    * try to create and/or check all needed files in the MetaInfo.
    *
-   * @exception IOException when creating and/or checking files fails.
+   * Does not check storage. Caller MUST call check()
    */
   public Storage(I2PSnarkUtil util, MetaInfo metainfo, StorageListener listener)
-    throws IOException
   {
     _util = util;
     _log = util.getContext().logManager().getLog(Storage.class);
     this.metainfo = metainfo;
     this.listener = listener;
     needed = metainfo.getPieces();
-    _probablyComplete = false;
     bitfield = new BitField(needed);
     piece_size = metainfo.getPieceLength(0);
     pieces = needed;
@@ -101,14 +111,18 @@ public class Storage
   }
 
   /**
-   * Creates a storage from the existing file or directory together
-   * with an appropriate MetaInfo file as can be announced on the
-   * given announce String location.
+   * Creates a storage from the existing file or directory.
+   * Creates an in-memory metainfo but does not save it to
+   * a file, caller must do that.
+   *
+   * Creates the metainfo, this may take a LONG time. BLOCKING.
    *
    * @param announce may be null
    * @param listener may be null
+   * @throws IOException when creating and/or checking files fails.
    */
   public Storage(I2PSnarkUtil util, File baseFile, String announce,
+                 List<List<String>> announce_list,
                  boolean privateTorrent, StorageListener listener)
     throws IOException
   {
@@ -129,10 +143,18 @@ public class Storage
 
     if (total <= 0)
         throw new IOException("Torrent contains no data");
+    if (total > MAX_TOTAL_SIZE)
+        throw new IOException("Torrent too big (" + total + " bytes), max is " + MAX_TOTAL_SIZE);
 
-    int pc_size = MIN_PIECE_SIZE;
+    int pc_size;
+    if (total <= 5*1024*1024)
+        pc_size = DEFAULT_PIECE_SIZE / 4;
+    else if (total <= 10*1024*1024)
+        pc_size = DEFAULT_PIECE_SIZE / 2;
+    else
+        pc_size = DEFAULT_PIECE_SIZE;
     int pcs = (int) ((total - 1)/pc_size) + 1;
-    while (pcs > MAX_PIECES && pc_size < MAX_PIECE_SIZE)
+    while (pcs > (MAX_PIECES * 2 / 3) && pc_size < MAX_PIECE_SIZE)
       {
         pc_size *= 2;
         pcs = (int) ((total - 1)/pc_size) +1;
@@ -164,9 +186,11 @@ public class Storage
         lengthsList = null;
       }
 
+    // TODO thread this so we can return and show something on the UI
     byte[] piece_hashes = fast_digestCreate();
     metainfo = new MetaInfo(announce, baseFile.getName(), null, files,
-                            lengthsList, piece_size, piece_hashes, total, privateTorrent);
+                            lengthsList, piece_size, piece_hashes, total, privateTorrent,
+                            announce_list);
 
   }
 
@@ -199,6 +223,8 @@ public class Storage
 
   private void getFiles(File base) throws IOException
   {
+    if (base.getAbsolutePath().equals("/"))
+        throw new IOException("Don't seed root");
     ArrayList files = new ArrayList();
     addFiles(files, base);
 
@@ -227,12 +253,15 @@ public class Storage
       }
   }
 
-  private void addFiles(List l, File f)
-  {
-    if (!f.isDirectory())
-      l.add(f);
-    else
-      {
+  /**
+   *  @throws IOException if too many total files
+   */
+  private void addFiles(List l, File f) throws IOException {
+    if (!f.isDirectory()) {
+        if (l.size() >= SnarkManager.MAX_FILES_PER_TORRENT)
+            throw new IOException("Too many files, limit is " + SnarkManager.MAX_FILES_PER_TORRENT + ", zip them?");
+        l.add(f);
+    } else {
         File[] files = f.listFiles();
         if (files == null)
           {
@@ -278,6 +307,23 @@ public class Storage
       return changed;
   }
 
+  /**
+   *  File checking in progress.
+   *  @since 0.9.3
+   */
+  public boolean isChecking() {
+      return _isChecking;
+  }
+
+  /**
+   *  Disk allocation (ballooning) in progress.
+   *  Always false on Windows.
+   *  @since 0.9.3
+   */
+  public boolean isAllocating() {
+      return _allocateCount.get() > 0;
+  }
+
   /**
    *  @param file canonical path (non-directory)
    *  @return number of bytes remaining; -1 if unknown file
@@ -697,11 +743,25 @@ public class Storage
    * This is called at the beginning, and at presumed completion,
    * so we have to be careful about locking.
    *
+   * TODO thread the checking so we can return and display
+   * something on the UI
+   *
    * @param recheck if true, this is a check after we downloaded the
    *        last piece, and we don't modify the global bitfield unless
    *        the check fails.
    */
-  private void checkCreateFiles(boolean recheck) throws IOException
+  private void checkCreateFiles(boolean recheck) throws IOException {
+      synchronized(this) {
+          _isChecking = true;
+          try {
+              locked_checkCreateFiles(recheck);
+          } finally {
+              _isChecking = false;
+          }
+      }
+  }
+
+  private void locked_checkCreateFiles(boolean recheck) throws IOException
   {
     // Whether we are resuming or not,
     // if any of the files already exists we assume we are resuming.
@@ -861,10 +921,19 @@ public class Storage
     final int ZEROBLOCKSIZE = (int) Math.min(remaining, 32*1024);
     byte[] zeros = new byte[ZEROBLOCKSIZE];
     rafs[nr].seek(0);
-    while (remaining > 0) {
-        int size = (int) Math.min(remaining, ZEROBLOCKSIZE);
-        rafs[nr].write(zeros, 0, size);
-        remaining -= size;
+    // don't bother setting flag for small files
+    if (remaining > 20*1024*1024)
+        _allocateCount.incrementAndGet();
+    try {
+        while (remaining > 0) {
+            int size = (int) Math.min(remaining, ZEROBLOCKSIZE);
+            rafs[nr].write(zeros, 0, size);
+            remaining -= size;
+        }
+    } finally {
+        remaining = lengths[nr];
+        if (remaining > 20*1024*1024)
+            _allocateCount.decrementAndGet();
     }
     isSparse[nr] = false;
   }
@@ -898,26 +967,34 @@ public class Storage
    * Returns a byte array containing a portion of the requested piece or null if
    * the storage doesn't contain the piece yet.
    */
-  public byte[] getPiece(int piece, int off, int len) throws IOException
+  public ByteArray getPiece(int piece, int off, int len) throws IOException
   {
     if (!bitfield.get(piece))
       return null;
 
     //Catch a common place for OOMs esp. on 1MB pieces
+    ByteArray rv;
     byte[] bs;
     try {
-      bs = new byte[len];
+        // Will be restored to cache in Message.sendMessage()
+        if (len == BUFSIZE)
+            rv = _cache.acquire();
+        else
+            rv = new ByteArray(new byte[len]);
     } catch (OutOfMemoryError oom) {
       if (_log.shouldLog(Log.WARN))
           _log.warn("Out of memory, can't honor request for piece " + piece, oom);
       return null;
     }
+    bs = rv.getData();
     getUncheckedPiece(piece, bs, off, len);
-    return bs;
+    return rv;
   }
 
   /**
    * Put the piece in the Storage if it is correct.
+   * Warning - takes a LONG time if complete as it does the recheck here.
+   * TODO thread the recheck?
    *
    * @return true if the piece was correct (sha metainfo hash
    * matches), otherwise false.
@@ -935,9 +1012,9 @@ public class Storage
           // TODO alternative - check hash on the fly as we write to the file,
           // to save another I/O pass
           boolean correctHash = metainfo.checkPiece(pp);
-          if (listener != null)
-            listener.storageChecked(this, piece, correctHash);
           if (!correctHash) {
+              if (listener != null)
+                  listener.storageChecked(this, piece, false);
               return false;
           }
 
@@ -999,6 +1076,9 @@ public class Storage
             complete = needed == 0;
           }
       }
+    // tell listener after counts are updated
+    if (listener != null)
+        listener.storageChecked(this, piece, true);
 
     if (complete) {
       // do we also need to close all of the files and reopen
@@ -1136,4 +1216,43 @@ public class Storage
     rafs[i] = null;
   }
 
+  /**
+   *  Create a metainfo.
+   *  Used in the installer build process; do not comment out.
+   *  @since 0.9.4
+   */
+  public static void main(String[] args) {
+      if (args.length < 1 || args.length > 2) {
+          System.err.println("Usage: Storage file-or-dir [announceURL]");
+          System.exit(1);
+      }
+      File base = new File(args[0]);
+      String announce = args.length == 2 ? args[1] : null;
+      I2PAppContext ctx = I2PAppContext.getGlobalContext();
+      I2PSnarkUtil util = new I2PSnarkUtil(ctx);
+      File file = null;
+      FileOutputStream out = null;
+      try {
+          Storage storage = new Storage(util, base, announce, null, false, null);
+          MetaInfo meta = storage.getMetaInfo();
+          file = new File(storage.getBaseName() + ".torrent");
+          out = new FileOutputStream(file);
+          out.write(meta.getTorrentData());
+          String hex = DataHelper.toString(meta.getInfoHash());
+          System.out.println("Created:     " + file);
+          System.out.println("InfoHash:    " + hex);
+          String basename = base.getName().replace(" ", "%20");
+          String magnet = MagnetURI.MAGNET_FULL + hex + "&dn=" + basename;
+          if (announce != null)
+              magnet += "&tr=" + announce;
+          System.out.println("Magnet:      " + magnet);
+      } catch (IOException ioe) {
+          if (file != null)
+              file.delete();
+          ioe.printStackTrace();
+          System.exit(1);
+      } finally {
+          try { if (out != null) out.close(); } catch (IOException ioe) {}
+      }
+  }
 }

apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java

@@ -20,6 +20,7 @@
 
 package org.klomp.snark;
 
+import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
@@ -30,6 +31,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
@@ -39,10 +41,12 @@ import java.util.Set;
 import net.i2p.I2PAppContext;
 import net.i2p.data.DataHelper;
 import net.i2p.data.Hash;
+import net.i2p.util.ConvertToHash;
 import net.i2p.util.I2PAppThread;
 import net.i2p.util.Log;
 import net.i2p.util.SimpleTimer2;
 
+import org.klomp.snark.bencode.InvalidBEncodingException;
 import org.klomp.snark.dht.DHT;
 
 /**
@@ -70,6 +74,8 @@ public class TrackerClient implements Runnable {
   private static final String COMPLETED_EVENT = "completed";
   private static final String STOPPED_EVENT = "stopped";
   private static final String NOT_REGISTERED  = "torrent not registered"; //bytemonsoon
+  /** this is our equivalent to router.utorrent.com for bootstrap */
+  private static final String DEFAULT_BACKUP_TRACKER = "http://tracker.welterde.i2p/a";
 
   private final static int SLEEP = 5; // 5 minutes.
   private final static int DELAY_MIN = 2000; // 2 secs.
@@ -78,7 +84,7 @@ public class TrackerClient implements Runnable {
   private final static int INITIAL_SLEEP = 90*1000;
   private final static int MAX_CONSEC_FAILS = 5;    // slow down after this
   private final static int LONG_SLEEP = 30*60*1000; // sleep a while after lots of fails
-  private final static long MIN_TRACKER_ANNOUNCE_INTERVAL = 10*60*1000;
+  private final static long MIN_TRACKER_ANNOUNCE_INTERVAL = 15*60*1000;
   private final static long MIN_DHT_ANNOUNCE_INTERVAL = 10*60*1000;
 
   private final I2PSnarkUtil _util;
@@ -105,7 +111,8 @@ public class TrackerClient implements Runnable {
   private boolean completed;
   private volatile boolean _fastUnannounce;
   private long lastDHTAnnounce;
-  private final List<Tracker> trackers;
+  private final List<TCTracker> trackers;
+  private final List<TCTracker> backupTrackers;
 
   /**
    * Call start() to start it.
@@ -131,6 +138,7 @@ public class TrackerClient implements Runnable {
     this.infoHash = urlencode(snark.getInfoHash());
     this.peerID = urlencode(snark.getID());
     this.trackers = new ArrayList(2);
+    this.backupTrackers = new ArrayList(2);
   }
 
   public synchronized void start() {
@@ -233,7 +241,7 @@ public class TrackerClient implements Runnable {
           if (!_initialized) {
               _initialized = true;
               // FIXME only when starting everybody at once, not for a single torrent
-              long delay = I2PAppContext.getGlobalContext().random().nextInt(30*1000);
+              long delay = _util.getContext().random().nextInt(30*1000);
               try {
                   Thread.sleep(delay);
               } catch (InterruptedException ie) {}
@@ -264,49 +272,90 @@ public class TrackerClient implements Runnable {
         primary = meta.getAnnounce();
     else if (additionalTrackerURL != null)
         primary = additionalTrackerURL;
+    Set<Hash> trackerHashes = new HashSet(8);
+
+    // primary tracker
     if (primary != null) {
-        if (isValidAnnounce(primary)) {
-            trackers.add(new Tracker(primary, true));
-            _log.debug("Announce: [" + primary + "] infoHash: " + infoHash);
+        if (isNewValidTracker(trackerHashes, primary)) {
+            trackers.add(new TCTracker(primary, true));
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("Announce: [" + primary + "] infoHash: " + infoHash);
         } else {
-            _log.warn("Skipping invalid or non-i2p announce: " + primary);
+            if (_log.shouldLog(Log.WARN))
+                _log.warn("Skipping invalid or non-i2p announce: " + primary);
         }
     } else {
         _log.warn("No primary announce");
-        primary = "";
     }
-    List tlist = _util.getOpenTrackers();
-    if (tlist != null && (meta == null || !meta.isPrivate())) {
+
+    // announce list
+    if (meta != null && !meta.isPrivate()) {
+        List<List<String>> list = meta.getAnnounceList();
+        if (list != null) {
+            for (List<String> llist : list) {
+                for (String url : llist) {
+                    if (!isNewValidTracker(trackerHashes, url))
+                        continue;
+                    trackers.add(new TCTracker(url, trackers.isEmpty()));
+                    if (_log.shouldLog(Log.DEBUG))
+                        _log.debug("Additional announce (list): [" + url + "] for infoHash: " + infoHash);
+                }
+            }
+        }
+    }
+
+    // configured open trackers
+    if (meta == null || !meta.isPrivate()) {
+        List<String> tlist = _util.getOpenTrackers();
         for (int i = 0; i < tlist.size(); i++) {
-             String url = (String)tlist.get(i);
-             if (!isValidAnnounce(url)) {
-                _log.error("Bad announce URL: [" + url + "]");
+            String url = tlist.get(i);
+            if (!isNewValidTracker(trackerHashes, url))
                 continue;
-             }
-             int slash = url.indexOf('/', 7);
-             if (slash <= 7) {
-                _log.error("Bad announce URL: [" + url + "]");
+            // opentrackers are primary if we don't have primary
+            trackers.add(new TCTracker(url, trackers.isEmpty()));
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("Additional announce: [" + url + "] for infoHash: " + infoHash);
+        }
+    }
+
+    // backup trackers if DHT needs bootstrapping
+    if (trackers.isEmpty() && (meta == null || !meta.isPrivate())) {
+        List<String> tlist = _util.getBackupTrackers();
+        for (int i = 0; i < tlist.size(); i++) {
+            String url = tlist.get(i);
+            if (!isNewValidTracker(trackerHashes, url))
                 continue;
-             }
-             if (primary.startsWith(url.substring(0, slash)))
-                continue;
-             String dest = _util.lookup(url.substring(7, slash));
-             if (dest == null) {
-                _log.error("Announce host unknown: [" + url.substring(7, slash) + "]");
-                continue;
-             }
-             if (primary.startsWith("http://" + dest))
-                continue;
-             if (primary.startsWith("http://i2p/" + dest))
-                continue;
-             // opentrackers are primary if we don't have primary
-             trackers.add(new Tracker(url, primary.equals("")));
-             _log.debug("Additional announce: [" + url + "] for infoHash: " + infoHash);
+            backupTrackers.add(new TCTracker(url, false));
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("Backup announce: [" + url + "] for infoHash: " + infoHash);
+        }
+        if (backupTrackers.isEmpty()) {
+            backupTrackers.add(new TCTracker(DEFAULT_BACKUP_TRACKER, false));
         }
     }
     this.completed = coordinator.getLeft() == 0;
   }
 
+  /**
+   *  @param existing the ones we already know about
+   *  @param ann an announce URL non-null
+   *  @return true if ann is valid and new; adds to existing if returns true
+   *  @since 0.9.5
+   */
+  private boolean isNewValidTracker(Set<Hash> existing, String ann) {
+      Hash h = getHostHash(ann);
+      if (h == null) {
+         _log.error("Bad announce URL: [" + ann + ']');
+         return false;
+      }
+      boolean rv = existing.add(h);
+      if (!rv) {
+          if (_log.shouldLog(Log.INFO))
+             _log.info("Dup announce URL: [" + ann + ']');
+      }
+      return rv;
+  }
+
   /**
    *  Announce to all the trackers, get peers from PEX and DHT, then queue up a SimpleTimer2 event.
    *  This will take several seconds to several minutes.
@@ -315,7 +364,7 @@ public class TrackerClient implements Runnable {
   private void loop() {
     try
       {
-        Random r = I2PAppContext.getGlobalContext().random();
+        // normally this will only go once, then call queueLoop() and return
         while(!stop)
           {
             if (!verifyConnected()) {
@@ -325,187 +374,25 @@ public class TrackerClient implements Runnable {
 
             // Local DHT tracker announce
             DHT dht = _util.getDHT();
-            if (dht != null)
+            if (dht != null && (meta == null || !meta.isPrivate()))
                 dht.announce(snark.getInfoHash());
 
-            long uploaded = coordinator.getUploaded();
-            long downloaded = coordinator.getDownloaded();
-            long left = coordinator.getLeft();   // -1 in magnet mode
-            
-            // First time we got a complete download?
-            String event;
-            if (!completed && left == 0)
-              {
-                completed = true;
-                event = COMPLETED_EVENT;
-              }
-            else
-              event = NO_EVENT;
-            
-            // *** loop once for each tracker
             int maxSeenPeers = 0;
-            for (Tracker tr : trackers) {
-              if ((!stop) && (!tr.stop) &&
-                  (completed || coordinator.needOutboundPeers() || !tr.started) &&
-                  (event.equals(COMPLETED_EVENT) || System.currentTimeMillis() > tr.lastRequestTime + tr.interval))
-              {
-                try
-                  {
-                    if (!tr.started)
-                      event = STARTED_EVENT;
-                    TrackerInfo info = doRequest(tr, infoHash, peerID,
-                                                 uploaded, downloaded, left,
-                                                 event);
-
-                    snark.setTrackerProblems(null);
-                    tr.trackerProblems = null;
-                    tr.registerFails = 0;
-                    tr.consecutiveFails = 0;
-                    if (tr.isPrimary)
-                        consecutiveFails = 0;
-                    runStarted = true;
-                    tr.started = true;
-
-                    Set<Peer> peers = info.getPeers();
-                    tr.seenPeers = info.getPeerCount();
-                    if (snark.getTrackerSeenPeers() < tr.seenPeers) // update rising number quickly
-                        snark.setTrackerSeenPeers(tr.seenPeers);
-
-                    // pass everybody over to our tracker
-                    dht = _util.getDHT();
-                    if (dht != null) {
-                        for (Peer peer : peers) {
-                            dht.announce(snark.getInfoHash(), peer.getPeerID().getDestHash());
-                        }
-                    }
-
-                    if (coordinator.needOutboundPeers()) {
-                        // we only want to talk to new people if we need things
-                        // from them (duh)
-                        List<Peer> ordered = new ArrayList(peers);
-                        Collections.shuffle(ordered, r);
-                        Iterator<Peer> it = ordered.iterator();
-                        while ((!stop) && it.hasNext() && coordinator.needOutboundPeers()) {
-                          Peer cur = it.next();
-                          // FIXME if id == us || dest == us continue;
-                          // only delay if we actually make an attempt to add peer
-                          if(coordinator.addPeer(cur) && it.hasNext()) {
-                            int delay = r.nextInt(DELAY_RAND) + DELAY_MIN;
-                            try { Thread.sleep(delay); } catch (InterruptedException ie) {}
-                          }
-                        }
-                    }
-                  }
-                catch (IOException ioe)
-                  {
-                    // Probably not fatal (if it doesn't last to long...)
-                    if (_log.shouldLog(Log.WARN))
-                        _log.warn
-                      ("WARNING: Could not contact tracker at '"
-                       + tr.announce + "': " + ioe);
-                    tr.trackerProblems = ioe.getMessage();
-                    // don't show secondary tracker problems to the user
-                    if (tr.isPrimary)
-                      snark.setTrackerProblems(tr.trackerProblems);
-                    if (tr.trackerProblems.toLowerCase(Locale.US).startsWith(NOT_REGISTERED)) {
-                      // Give a guy some time to register it if using opentrackers too
-                      if (trackers.size() == 1) {
-                        stop = true;
-                        snark.stopTorrent();
-                      } else { // hopefully each on the opentrackers list is really open
-                        if (tr.registerFails++ > MAX_REGISTER_FAILS)
-                          tr.stop = true;
-                      }
-                    }
-                    if (++tr.consecutiveFails == MAX_CONSEC_FAILS) {
-                        tr.seenPeers = 0;
-                        if (tr.interval < LONG_SLEEP)
-                            tr.interval = LONG_SLEEP;  // slow down
-                    }
-                  }
-              } else {
-                  if (_log.shouldLog(Log.INFO))
-                      _log.info("Not announcing to " + tr.announce + " last announce was " +
-                               new Date(tr.lastRequestTime) + " interval is " + DataHelper.formatDuration(tr.interval));
-              }
-              if ((!tr.stop) && maxSeenPeers < tr.seenPeers)
-                  maxSeenPeers = tr.seenPeers;
-            }  // *** end of trackers loop here
-
-            // Get peers from PEX
-            if (coordinator.needOutboundPeers() && (meta == null || !meta.isPrivate()) && !stop) {
-                Set<PeerID> pids = coordinator.getPEXPeers();
-                if (!pids.isEmpty()) {
-                    if (_log.shouldLog(Log.INFO))
-                        _log.info("Got " + pids.size() + " from PEX");
-                    List<Peer> peers = new ArrayList(pids.size());
-                    for (PeerID pID : pids) {
-                        peers.add(new Peer(pID, snark.getID(), snark.getInfoHash(), snark.getMetaInfo()));
-                    }
-                    Collections.shuffle(peers, r);
-                    Iterator<Peer> it = peers.iterator();
-                    while ((!stop) && it.hasNext() && coordinator.needOutboundPeers()) {
-                        Peer cur = it.next();
-                        if (coordinator.addPeer(cur) && it.hasNext()) {
-                            int delay = r.nextInt(DELAY_RAND) + DELAY_MIN;
-                            try { Thread.sleep(delay); } catch (InterruptedException ie) {}
-                         }
-                    }
-                }
-            } else {
-                if (_log.shouldLog(Log.INFO))
-                    _log.info("Not getting PEX peers");
+            if (!trackers.isEmpty())
+                maxSeenPeers = getPeersFromTrackers(trackers);
+            int p = getPeersFromPEX();
+            if (p > maxSeenPeers)
+                maxSeenPeers = p;
+            p = getPeersFromDHT();
+            if (p > maxSeenPeers)
+                maxSeenPeers = p;
+            // backup if DHT needs bootstrapping
+            if (trackers.isEmpty() && !backupTrackers.isEmpty() && dht != null && dht.size() < 16) {
+                p = getPeersFromTrackers(backupTrackers);
+                if (p > maxSeenPeers)
+                    maxSeenPeers = p;
             }
 
-            // Get peers from DHT
-            // FIXME this needs to be in its own thread
-            dht = _util.getDHT();
-            if (dht != null && (meta == null || !meta.isPrivate()) && (!stop) &&
-                _util.getContext().clock().now() >  lastDHTAnnounce + MIN_DHT_ANNOUNCE_INTERVAL) {
-                int numwant;
-                if (event.equals(STOPPED_EVENT) || !coordinator.needOutboundPeers())
-                    numwant = 1;
-                else
-                    numwant = _util.getMaxConnections();
-                Collection<Hash> hashes = dht.getPeers(snark.getInfoHash(), numwant, 2*60*1000);
-                if (!hashes.isEmpty()) {
-                    runStarted = true;
-                    lastDHTAnnounce = _util.getContext().clock().now();
-                }
-                if (_log.shouldLog(Log.INFO))
-                    _log.info("Got " + hashes + " from DHT");
-                // announce  ourselves while the token is still good
-                // FIXME this needs to be in its own thread
-                if (!stop) {
-                    // announce only to the 1 closest
-                    int good = dht.announce(snark.getInfoHash(), 1, 5*60*1000);
-                    if (_log.shouldLog(Log.INFO))
-                        _log.info("Sent " + good + " good announces to DHT");
-                }
-
-                // now try these peers
-                if ((!stop) && !hashes.isEmpty()) {
-                    List<Peer> peers = new ArrayList(hashes.size());
-                    for (Hash h : hashes) {
-                        PeerID pID = new PeerID(h.getData(), _util);
-                        peers.add(new Peer(pID, snark.getID(), snark.getInfoHash(), snark.getMetaInfo()));
-                    }
-                    Collections.shuffle(peers, r);
-                    Iterator<Peer> it = peers.iterator();
-                    while ((!stop) && it.hasNext() && coordinator.needOutboundPeers()) {
-                        Peer cur = it.next();
-                        if (coordinator.addPeer(cur) && it.hasNext()) {
-                            int delay = r.nextInt(DELAY_RAND) + DELAY_MIN;
-                            try { Thread.sleep(delay); } catch (InterruptedException ie) {}
-                         }
-                    }
-                }
-            } else {
-                if (_log.shouldLog(Log.INFO))
-                    _log.info("Not getting DHT peers");
-            }
-
-
             // we could try and total the unique peers but that's too hard for now
             snark.setTrackerSeenPeers(maxSeenPeers);
 
@@ -516,6 +403,7 @@ public class TrackerClient implements Runnable {
                 // Sleep some minutes...
                 // Sleep the minimum interval for all the trackers, but 60s minimum
                 int delay;
+                Random r = _util.getContext().random();
                 int random = r.nextInt(120*1000);
                 if (completed && runStarted)
                   delay = 3*SLEEP*60*1000 + random;
@@ -547,6 +435,219 @@ public class TrackerClient implements Runnable {
       }
   }
 
+  /**
+   *  @return max peers seen
+   */
+  private int getPeersFromTrackers(List<TCTracker> trckrs) {
+            long left = coordinator.getLeft();   // -1 in magnet mode
+            
+            // First time we got a complete download?
+            boolean newlyCompleted;
+            if (!completed && left == 0) {
+                completed = true;
+                newlyCompleted = true;
+            } else {
+                newlyCompleted = false;
+            }
+
+            // *** loop once for each tracker
+            int maxSeenPeers = 0;
+            for (TCTracker tr : trckrs) {
+              if ((!stop) && (!tr.stop) &&
+                  (completed || coordinator.needOutboundPeers() || !tr.started) &&
+                  (newlyCompleted || System.currentTimeMillis() > tr.lastRequestTime + tr.interval))
+              {
+                try
+                  {
+                    long uploaded = coordinator.getUploaded();
+                    long downloaded = coordinator.getDownloaded();
+                    left = coordinator.getLeft();
+                    String event;
+                    if (!tr.started) {
+                        event = STARTED_EVENT;
+                    } else if (newlyCompleted) {
+                        event = COMPLETED_EVENT;
+                    } else {
+                        event = NO_EVENT;
+                    }
+                    TrackerInfo info = doRequest(tr, infoHash, peerID,
+                                                 uploaded, downloaded, left,
+                                                 event);
+
+                    snark.setTrackerProblems(null);
+                    tr.trackerProblems = null;
+                    tr.registerFails = 0;
+                    tr.consecutiveFails = 0;
+                    if (tr.isPrimary)
+                        consecutiveFails = 0;
+                    runStarted = true;
+                    tr.started = true;
+
+                    Set<Peer> peers = info.getPeers();
+                    tr.seenPeers = info.getPeerCount();
+                    if (snark.getTrackerSeenPeers() < tr.seenPeers) // update rising number quickly
+                        snark.setTrackerSeenPeers(tr.seenPeers);
+
+                    // pass everybody over to our tracker
+                    DHT dht = _util.getDHT();
+                    if (dht != null) {
+                        for (Peer peer : peers) {
+                            dht.announce(snark.getInfoHash(), peer.getPeerID().getDestHash());
+                        }
+                    }
+
+                    if (coordinator.needOutboundPeers()) {
+                        // we only want to talk to new people if we need things
+                        // from them (duh)
+                        List<Peer> ordered = new ArrayList(peers);
+                        Random r = _util.getContext().random();
+                        Collections.shuffle(ordered, r);
+                        Iterator<Peer> it = ordered.iterator();
+                        while ((!stop) && it.hasNext() && coordinator.needOutboundPeers()) {
+                          Peer cur = it.next();
+                          // FIXME if id == us || dest == us continue;
+                          // only delay if we actually make an attempt to add peer
+                          if(coordinator.addPeer(cur) && it.hasNext()) {
+                            int delay = r.nextInt(DELAY_RAND) + DELAY_MIN;
+                            try { Thread.sleep(delay); } catch (InterruptedException ie) {}
+                          }
+                        }
+                    }
+                  }
+                catch (IOException ioe)
+                  {
+                    // Probably not fatal (if it doesn't last to long...)
+                    if (_log.shouldLog(Log.WARN))
+                        _log.warn
+                      ("WARNING: Could not contact tracker at '"
+                       + tr.announce + "': " + ioe);
+                    tr.trackerProblems = ioe.getMessage();
+                    // don't show secondary tracker problems to the user
+                    if (tr.isPrimary)
+                      snark.setTrackerProblems(tr.trackerProblems);
+                    if (tr.trackerProblems.toLowerCase(Locale.US).startsWith(NOT_REGISTERED)) {
+                      // Give a guy some time to register it if using opentrackers too
+                      //if (trckrs.size() == 1) {
+                      //  stop = true;
+                      //  snark.stopTorrent();
+                      //} else { // hopefully each on the opentrackers list is really open
+                        if (tr.registerFails++ > MAX_REGISTER_FAILS)
+                          tr.stop = true;
+                      //
+                    }
+                    if (++tr.consecutiveFails == MAX_CONSEC_FAILS) {
+                        tr.seenPeers = 0;
+                        if (tr.interval < LONG_SLEEP)
+                            tr.interval = LONG_SLEEP;  // slow down
+                    }
+                  }
+              } else {
+                  if (_log.shouldLog(Log.INFO))
+                      _log.info("Not announcing to " + tr.announce + " last announce was " +
+                               new Date(tr.lastRequestTime) + " interval is " + DataHelper.formatDuration(tr.interval));
+              }
+              if ((!tr.stop) && maxSeenPeers < tr.seenPeers)
+                  maxSeenPeers = tr.seenPeers;
+            }  // *** end of trackers loop here
+
+            return maxSeenPeers;
+  }
+
+  /**
+   *  @return max peers seen
+   */
+  private int getPeersFromPEX() {
+            // Get peers from PEX
+            int rv = 0;
+            if (coordinator.needOutboundPeers() && (meta == null || !meta.isPrivate()) && !stop) {
+                Set<PeerID> pids = coordinator.getPEXPeers();
+                if (!pids.isEmpty()) {
+                    if (_log.shouldLog(Log.INFO))
+                        _log.info("Got " + pids.size() + " from PEX");
+                    List<Peer> peers = new ArrayList(pids.size());
+                    for (PeerID pID : pids) {
+                        peers.add(new Peer(pID, snark.getID(), snark.getInfoHash(), snark.getMetaInfo()));
+                    }
+                    Random r = _util.getContext().random();
+                    Collections.shuffle(peers, r);
+                    Iterator<Peer> it = peers.iterator();
+                    while ((!stop) && it.hasNext() && coordinator.needOutboundPeers()) {
+                        Peer cur = it.next();
+                        if (coordinator.addPeer(cur) && it.hasNext()) {
+                            int delay = r.nextInt(DELAY_RAND) + DELAY_MIN;
+                            try { Thread.sleep(delay); } catch (InterruptedException ie) {}
+                         }
+                    }
+                    rv = pids.size();
+                    pids.clear();
+                }
+            } else {
+                if (_log.shouldLog(Log.INFO))
+                    _log.info("Not getting PEX peers");
+            }
+            return rv;
+    }
+
+  /**
+   *  @return max peers seen
+   */
+  private int getPeersFromDHT() {
+            // Get peers from DHT
+            // FIXME this needs to be in its own thread
+            int rv = 0;
+            DHT dht = _util.getDHT();
+            if (dht != null && (meta == null || !meta.isPrivate()) && (!stop) &&
+                _util.getContext().clock().now() >  lastDHTAnnounce + MIN_DHT_ANNOUNCE_INTERVAL) {
+                int numwant;
+                if (!coordinator.needOutboundPeers())
+                    numwant = 1;
+                else
+                    numwant = _util.getMaxConnections();
+                Collection<Hash> hashes = dht.getPeers(snark.getInfoHash(), numwant, 2*60*1000);
+                if (!hashes.isEmpty()) {
+                    runStarted = true;
+                    lastDHTAnnounce = _util.getContext().clock().now();
+                    rv = hashes.size();
+                }
+                if (_log.shouldLog(Log.INFO))
+                    _log.info("Got " + hashes + " from DHT");
+                // announce  ourselves while the token is still good
+                // FIXME this needs to be in its own thread
+                if (!stop) {
+                    // announce only to the 1 closest
+                    int good = dht.announce(snark.getInfoHash(), 1, 5*60*1000);
+                    if (_log.shouldLog(Log.INFO))
+                        _log.info("Sent " + good + " good announces to DHT");
+                }
+
+                // now try these peers
+                if ((!stop) && !hashes.isEmpty()) {
+                    List<Peer> peers = new ArrayList(hashes.size());
+                    for (Hash h : hashes) {
+                        try {
+                            PeerID pID = new PeerID(h.getData(), _util);
+                            peers.add(new Peer(pID, snark.getID(), snark.getInfoHash(), snark.getMetaInfo()));
+                        } catch (InvalidBEncodingException ibe) {}
+                    }
+                    Random r = _util.getContext().random();
+                    Collections.shuffle(peers, r);
+                    Iterator<Peer> it = peers.iterator();
+                    while ((!stop) && it.hasNext() && coordinator.needOutboundPeers()) {
+                        Peer cur = it.next();
+                        if (coordinator.addPeer(cur) && it.hasNext()) {
+                            int delay = r.nextInt(DELAY_RAND) + DELAY_MIN;
+                            try { Thread.sleep(delay); } catch (InterruptedException ie) {}
+                         }
+                    }
+                }
+            } else {
+                if (_log.shouldLog(Log.INFO))
+                    _log.info("Not getting DHT peers");
+            }
+            return rv;
+  }
+
+
   /**
    *  Creates a thread for each tracker in parallel if tunnel is still open
    *  @since 0.9.1
@@ -557,7 +658,7 @@ public class TrackerClient implements Runnable {
       if (dht != null)
           dht.unannounce(snark.getInfoHash());
       int i = 0;
-      for (Tracker tr : trackers) {
+      for (TCTracker tr : trackers) {
           if (_util.connected() &&
               tr.started && (!tr.stop) && tr.trackerProblems == null) {
               try {
@@ -577,9 +678,9 @@ public class TrackerClient implements Runnable {
    *  @since 0.9.1
    */
   private class Unannouncer implements Runnable {
-     private final Tracker tr;
+     private final TCTracker tr;
 
-     public Unannouncer(Tracker tr) {
+     public Unannouncer(TCTracker tr) {
          this.tr = tr;
      }
 
@@ -603,7 +704,7 @@ public class TrackerClient implements Runnable {
      }
   }
   
-  private TrackerInfo doRequest(Tracker tr, String infoHash,
+  private TrackerInfo doRequest(TCTracker tr, String infoHash,
                                 String peerID, long uploaded,
                                 long downloaded, long left, String event)
     throws IOException
@@ -630,7 +731,8 @@ public class TrackerClient implements Runnable {
     if (! event.equals(NO_EVENT))
         buf.append("&event=").append(event);
     buf.append("&numwant=");
-    if (left == 0 || event.equals(STOPPED_EVENT) || !coordinator.needOutboundPeers())
+    boolean small = left == 0 || event.equals(STOPPED_EVENT) || !coordinator.needOutboundPeers();
+    if (small)
         buf.append('0');
     else
         buf.append(_util.getMaxConnections());
@@ -641,14 +743,12 @@ public class TrackerClient implements Runnable {
     tr.lastRequestTime = System.currentTimeMillis();
     // Don't wait for a response to stopped when shutting down
     boolean fast = _fastUnannounce && event.equals(STOPPED_EVENT);
-    File fetched = _util.get(s, true, fast ? -1 : 0);
+    byte[] fetched = _util.get(s, true, fast ? -1 : 0, small ? 128 : 1024, small ? 1024 : 8*1024);
     if (fetched == null) {
         throw new IOException("Error fetching " + s);
     }
     
-    InputStream in = null;
-    try {
-        in = new FileInputStream(fetched);
+        InputStream in = new ByteArrayInputStream(fetched);
 
         TrackerInfo info = new TrackerInfo(in, snark.getID(),
                                            snark.getInfoHash(), snark.getMetaInfo(), _util);
@@ -661,10 +761,6 @@ public class TrackerClient implements Runnable {
 
         tr.interval = Math.max(MIN_TRACKER_ANNOUNCE_INTERVAL, info.getInterval() * 1000l);
         return info;
-    } finally {
-        if (in != null) try { in.close(); } catch (IOException ioe) {}
-        fetched.delete();
-    }
   }
 
   /**
@@ -698,6 +794,7 @@ public class TrackerClient implements Runnable {
   }
 
   /**
+   *  @param ann an announce URL
    *  @return true for i2p hosts only
    *  @since 0.7.12
    */
@@ -713,10 +810,38 @@ public class TrackerClient implements Runnable {
            url.getPort() < 0;
   }
 
-  private static class Tracker
+  /**
+   *  @param ann an announce URL non-null
+   *  @return a Hash for i2p hosts only, null otherwise
+   *  @since 0.9.5
+   */
+  private static Hash getHostHash(String ann) {
+    URL url;
+    try {
+        url = new URL(ann);
+    } catch (MalformedURLException mue) {
+        return null;
+    }
+    if (url.getPort() >= 0 || !url.getProtocol().equals("http"))
+        return null;
+    String host = url.getHost();
+    if (host.endsWith(".i2p"))
+        return ConvertToHash.getHash(host);
+    if (host.equals("i2p")) {
+        String path = url.getPath();
+        if (path == null || path.length() < 517 ||
+            !path.startsWith("/"))
+            return null;
+        String[] parts = path.substring(1).split("/?&;", 2);
+        return ConvertToHash.getHash(parts[0]);
+    }
+    return null;
+  }
+
+  private static class TCTracker
   {
-      String announce;
-      boolean isPrimary;
+      final String announce;
+      final boolean isPrimary;
       long interval;
       long lastRequestTime;
       String trackerProblems;
@@ -726,7 +851,7 @@ public class TrackerClient implements Runnable {
       int consecutiveFails;
       int seenPeers;
 
-      public Tracker(String a, boolean p)
+      public TCTracker(String a, boolean p)
       {
           announce = a;
           isPrimary = p;

apps/i2psnark/java/src/org/klomp/snark/UpdateHandler.java

@@ -0,0 +1,52 @@
+package org.klomp.snark;
+
+import java.net.URI;
+import java.util.List;
+
+import net.i2p.I2PAppContext;
+import net.i2p.update.*;
+
+/**
+ * <p>Handles the request to update the router by firing up a magnet.
+ * {@link net.i2p.util.EepGet} calls to download the latest signed update file
+ * and displaying the status to anyone who asks.
+ * </p>
+ * <p>After the download completes the signed update file is verified with
+ * {@link net.i2p.crypto.TrustedUpdate}, and if it's authentic the payload
+ * of the signed update file is unpacked and the router is restarted to complete
+ * the update process.
+ * </p>
+ *
+ * This does not do any checking, that is handled by the NewsFetcher.
+ *
+ * @since 0.9.4
+ */
+class UpdateHandler implements Updater {
+    private final I2PAppContext _context;
+    private final UpdateManager _umgr;
+    private final SnarkManager _smgr;
+    
+    public UpdateHandler(I2PAppContext ctx, UpdateManager umgr, SnarkManager smgr) {
+        _context = ctx;
+        _umgr = umgr;
+        _smgr = smgr;
+    }
+    
+    /**
+     *  Start a download and return a handle to the download task.
+     *  Should not block.
+     *
+     *  @param id plugin name or ignored
+     *  @param maxTime how long you have
+     *  @return active task or null if unable to download
+     */
+    public UpdateTask update(UpdateType type, UpdateMethod method, List<URI> updateSources,
+                             String id, String newVersion, long maxTime) {
+        if (type != UpdateType.ROUTER_SIGNED ||
+            method != UpdateMethod.TORRENT || updateSources.isEmpty())
+            return null;
+        UpdateRunner update = new UpdateRunner(_context, _umgr, _smgr, updateSources, newVersion);
+        _umgr.notifyProgress(update, "<b>" + _smgr.util().getString("Updating") + "</b>");
+        return update;
+    }
+}

apps/i2psnark/java/src/org/klomp/snark/UpdateRunner.java

@@ -0,0 +1,302 @@
+package org.klomp.snark;
+
+import java.io.File;
+import java.net.URI;
+import java.util.List;
+
+import net.i2p.I2PAppContext;
+import net.i2p.crypto.TrustedUpdate;
+import net.i2p.data.DataHelper;
+import net.i2p.update.*;
+import net.i2p.util.Log;
+import net.i2p.util.SimpleTimer2;
+import net.i2p.util.VersionComparator;
+
+/**
+ *  The downloader for router signed updates.
+ *
+ *  @since 0.9.4
+ */
+class UpdateRunner implements UpdateTask, CompleteListener {
+    private final I2PAppContext _context;
+    private final Log _log;
+    private final UpdateManager _umgr;
+    private final SnarkManager _smgr;
+    private final List<URI> _urls;
+    private volatile boolean _isRunning;
+    private volatile boolean _hasMetaInfo;
+    private volatile boolean _isComplete;
+    private final String _newVersion;
+    private URI _currentURI;
+    private Snark _snark;
+
+    private static final long MAX_LENGTH = 30*1024*1024;
+    private static final long METAINFO_TIMEOUT = 30*60*1000;
+    private static final long COMPLETE_TIMEOUT = 3*60*60*1000;
+    private static final long CHECK_INTERVAL = 3*60*1000;
+
+    public UpdateRunner(I2PAppContext ctx, UpdateManager umgr, SnarkManager smgr,
+                        List<URI> uris, String newVersion) { 
+        _context = ctx;
+        _log = ctx.logManager().getLog(getClass());
+        _umgr = umgr;
+        _smgr = smgr;
+        _urls = uris;
+        _newVersion = newVersion;
+    }
+
+    //////// begin UpdateTask methods
+
+    public boolean isRunning() { return _isRunning; }
+
+    public void shutdown() {
+        _isRunning = false;
+        if (_snark != null) {
+
+        }
+    }
+
+    public UpdateType getType() { return UpdateType.ROUTER_SIGNED; }
+
+    public UpdateMethod getMethod() { return UpdateMethod.TORRENT; }
+
+    public URI getURI() { return _currentURI; }
+
+    public String getID() { return ""; }
+
+    //////// end UpdateTask methods
+
+    public void start() {
+        _isRunning = true;
+        update();
+    }
+
+    /**
+     *  Loop through the entire list of update URLs.
+     *  For each one, first get the version from the first 56 bytes and see if
+     *  it is newer than what we are running now.
+     *  If it is, get the whole thing.
+     */
+    private void update() {
+        for (URI uri : _urls) {
+            _currentURI = uri;
+            String updateURL = uri.toString();
+            try {
+                MagnetURI magnet = new MagnetURI(_smgr.util(), updateURL);
+                byte[] ih = magnet.getInfoHash();
+                // do we already have it?
+                _snark = _smgr.getTorrentByInfoHash(ih);
+                if (_snark != null) {
+                    if (_snark.getMetaInfo() != null) {
+                         _hasMetaInfo = true;
+                         Storage storage = _snark.getStorage();
+                         if (storage != null && storage.complete())
+                             processComplete(_snark);
+                    }
+                    if (!_isComplete) {
+                        if (_snark.isStopped() && !_snark.isStarting())
+                            _snark.startTorrent();
+                        // we aren't a listener so we must poll
+                        new Watcher();
+                    }
+                    break;
+                }
+                String name = magnet.getName();
+                String trackerURL = magnet.getTrackerURL();
+                if (trackerURL == null && !_smgr.util().shouldUseDHT() &&
+                    !_smgr.util().shouldUseOpenTrackers()) {
+                    // but won't we use OT as a failsafe even if disabled?
+                    _umgr.notifyAttemptFailed(this, "No tracker, no DHT, no OT", null);
+                    continue;
+                }
+                _snark = _smgr.addMagnet(name, ih, trackerURL, true, true, this);
+                if (_snark != null) {
+                    updateStatus("<b>" + _smgr.util().getString("Updating from {0}", updateURL) + "</b>");
+                    new Timeout();
+                    break;
+                }
+            } catch (IllegalArgumentException iae) {
+                _log.error("Invalid update URL", iae);
+            }
+        }
+        if (_snark == null)
+            fatal("No valid URLs");
+    }
+
+    /**
+     *  This will run twice, once at the metainfo timeout and
+     *  once at the complete timeout.
+     */
+    private class Timeout extends SimpleTimer2.TimedEvent {
+        private final long _start = _context.clock().now();
+
+        public Timeout() {
+            super(_context.simpleTimer2(), METAINFO_TIMEOUT);
+        }
+
+        public void timeReached() {
+            if (_isComplete || !_isRunning)
+                return;
+            if (!_hasMetaInfo) {
+                fatal("Metainfo timeout");
+                return;
+            }
+            if (_context.clock().now() - _start >= COMPLETE_TIMEOUT) {
+                fatal("Complete timeout");
+                return;
+            }
+            reschedule(COMPLETE_TIMEOUT - METAINFO_TIMEOUT);
+        }
+    }
+
+    /**
+     *  Rarely used - only if the user added the torrent, so
+     *  we aren't a complete listener.
+     *  This will periodically until the complete timeout.
+     */
+    private class Watcher extends SimpleTimer2.TimedEvent {
+        private final long _start = _context.clock().now();
+
+        public Watcher() {
+            super(_context.simpleTimer2(), CHECK_INTERVAL);
+        }
+
+        public void timeReached() {
+            if (_hasMetaInfo && _snark.getRemainingLength() == 0 && !_isComplete)
+                processComplete(_snark);
+            if (_isComplete || !_isRunning)
+                return;
+            if (_context.clock().now() - _start >= METAINFO_TIMEOUT && !_hasMetaInfo) {
+                fatal("Metainfo timeout");
+                return;
+            }
+            if (_context.clock().now() - _start >= COMPLETE_TIMEOUT) {
+                fatal("Complete timeout");
+                return;
+            }
+            notifyProgress();
+            reschedule(CHECK_INTERVAL);
+        }
+    }
+
+    private void fatal(String error) {
+            if (_snark != null) {
+                if (_hasMetaInfo) {
+                    _smgr.stopTorrent(_snark, true);
+                    String file = _snark.getName();
+                    _smgr.removeTorrent(file);
+                    // delete torrent file
+                    File f = new File(_smgr.getDataDir(), file);
+                    f.delete();
+                    // delete data
+                    file = _snark.getBaseName();
+                    f = new File(_smgr.getDataDir(), file);
+                    f.delete();
+                } else {
+                    _smgr.deleteMagnet(_snark);
+                }
+            }
+            _umgr.notifyTaskFailed(this, error, null);
+            _log.error(error);
+            _isRunning = false;
+            // stop the tunnel if we were the only one running
+            if (_smgr.util().connected() && !_smgr.util().isConnecting()) {
+                for (Snark s : _smgr.getTorrents()) {
+                    if (!s.isStopped())
+                        return;
+                }
+                _smgr.util().disconnect();
+            }
+    }
+
+    private void processComplete(Snark snark) {
+        String dataFile = snark.getBaseName();
+        File f = new File(_smgr.getDataDir(), dataFile);
+        String sudVersion = TrustedUpdate.getVersionString(f);
+        if (_newVersion.equals(sudVersion))
+            _umgr.notifyComplete(this, _newVersion, f);
+        else
+            fatal("version mismatch");
+        _isComplete = true;
+    }
+
+    private void notifyProgress() {
+        if (_hasMetaInfo) {
+            long total = _snark.getTotalLength();
+            long remaining = _snark.getRemainingLength(); 
+            String status = "<b>" + _smgr.util().getString("Updating") + "</b>";
+            _umgr.notifyProgress(this, status, total - remaining, total);
+        }
+    }
+
+    //////// begin CompleteListener methods
+    //////// all pass through to SnarkManager
+
+    public void torrentComplete(Snark snark) {
+        processComplete(snark);
+        _smgr.torrentComplete(snark);
+    }
+
+    /**
+     *  This is called by stopTorrent() among others
+     */
+    public void updateStatus(Snark snark) {
+        if (snark.isStopped()) {
+            if (!_isComplete)
+                fatal("stopped by user");
+        }
+        _smgr.updateStatus(snark);
+    }
+
+    public String gotMetaInfo(Snark snark) {
+        MetaInfo info = snark.getMetaInfo();
+        if (info.getFiles() != null) {
+            fatal("more than 1 file");
+            return null;
+        }
+        if (info.isPrivate()) {
+            fatal("private torrent");
+            return null;
+        }
+        if (info.getTotalLength() > MAX_LENGTH) {
+            fatal("too big");
+            return null;
+        }
+        _hasMetaInfo = true;
+        notifyProgress();
+        return _smgr.gotMetaInfo(snark);
+    }
+
+    public void fatal(Snark snark, String error) {
+         fatal(error);
+        _smgr.fatal(snark, error);
+    }
+
+    public void addMessage(Snark snark, String message) {
+        _smgr.addMessage(snark, message);
+    }
+
+    public void gotPiece(Snark snark) {
+        notifyProgress();
+        _smgr.gotPiece(snark);
+    }
+
+    public long getSavedTorrentTime(Snark snark) {
+        return _smgr.getSavedTorrentTime(snark);
+    }
+
+    public BitField getSavedTorrentBitField(Snark snark) {
+        return _smgr.getSavedTorrentBitField(snark);
+    }
+
+    //////// end CompleteListener methods
+
+    private void updateStatus(String s) {
+        _umgr.notifyProgress(this, s);
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getName() + ' ' + getType() + ' ' + getID() + ' ' + getMethod() + ' ' + getURI();
+    }
+}

apps/i2psnark/java/src/org/klomp/snark/dht/DHTNodes.java

@@ -69,6 +69,9 @@ class DHTNodes {
 
     // begin ConcurrentHashMap methods
 
+    /**
+     *  @return known nodes, not total net size
+     */
     public int size() {
         return _nodeMap.size();
     }
@@ -86,8 +89,13 @@ class DHTNodes {
      *  @return the old value if present, else null
      */
     public NodeInfo putIfAbsent(NodeInfo nInfo) {
-        _kad.add(nInfo.getNID());
-        return _nodeMap.putIfAbsent(nInfo.getNID(), nInfo);
+        NodeInfo rv = _nodeMap.putIfAbsent(nInfo.getNID(), nInfo);
+        // ensure same object in both places
+        if (rv != null)
+            _kad.add(rv.getNID());
+        else
+            _kad.add(nInfo.getNID());
+        return rv;
     }
 
     public NodeInfo remove(NID nid) {
@@ -128,11 +136,19 @@ class DHTNodes {
         return _kad.getExploreKeys(MAX_BUCKET_AGE);
     }
 
+    /**
+     * Debug info, HTML formatted
+     * @since 0.9.4
+     */
+    public void renderStatusHTML(StringBuilder buf) {
+        buf.append(_kad.toString().replace("\n", "<br>\n"));
+    }
+
     /** */
     private class Cleaner extends SimpleTimer2.TimedEvent {
 
         public Cleaner() {
-            super(SimpleTimer2.getInstance(), CLEAN_TIME);
+            super(SimpleTimer2.getInstance(), 5 * CLEAN_TIME);
         }
 
         public void timeReached() {

apps/i2psnark/java/src/org/klomp/snark/dht/DHTTracker.java

@@ -39,6 +39,8 @@ class DHTTracker {
     private static final long DELTA_EXPIRE_TIME = 3*60*1000;
     private static final int MAX_PEERS = 2000;
     private static final int MAX_PEERS_PER_TORRENT = 150;
+    private static final int ABSOLUTE_MAX_PER_TORRENT = MAX_PEERS_PER_TORRENT * 2;
+    private static final int MAX_TORRENTS = 400;
 
     DHTTracker(I2PAppContext ctx) {
         _context = ctx;
@@ -62,17 +64,29 @@ class DHTTracker {
             _log.debug("Announce " + hash + " for " + ih);
         Peers peers = _torrents.get(ih);
         if (peers == null) {
+            if (_torrents.size() >= MAX_TORRENTS)
+                return;
             peers = new Peers();
             Peers peers2 = _torrents.putIfAbsent(ih, peers);
             if (peers2 != null)
                 peers = peers2;
         }
 
-        Peer peer = new Peer(hash.getData());
-        Peer peer2 = peers.putIfAbsent(peer, peer);
-        if (peer2 != null)
-            peer = peer2;
-        peer.setLastSeen(_context.clock().now());
+        if (peers.size() < ABSOLUTE_MAX_PER_TORRENT) {
+            Peer peer = new Peer(hash.getData());
+            Peer peer2 = peers.putIfAbsent(peer, peer);
+            if (peer2 != null)
+                peer = peer2;
+            peer.setLastSeen(_context.clock().now());
+        } else {
+            // We could update setLastSeen if he is already
+            // in there, but that would tend to keep
+            // the same set of peers.
+            // So let it expire so new ones can come in.
+            //Peer peer = peers.get(hash);
+            //if (peer != null)
+            //    peer.setLastSeen(_context.clock().now());
+        }
     }
 
     void unannounce(InfoHash ih, Hash hash) {
@@ -113,7 +127,7 @@ class DHTTracker {
     private class Cleaner extends SimpleTimer2.TimedEvent {
 
         public Cleaner() {
-            super(SimpleTimer2.getInstance(), CLEAN_TIME);
+            super(SimpleTimer2.getInstance(), 2 * CLEAN_TIME);
         }
 
         public void timeReached() {
@@ -122,6 +136,7 @@ class DHTTracker {
             long now = _context.clock().now();
             int torrentCount = 0;
             int peerCount = 0;
+            boolean tooMany = false;
             for (Iterator<Peers> iter = _torrents.values().iterator(); iter.hasNext(); ) {
                 Peers p = iter.next();
                 int recent = 0;
@@ -136,6 +151,7 @@ class DHTTracker {
                 }
                 if (recent > MAX_PEERS_PER_TORRENT) {
                     // too many, delete at random
+                    // TODO sort and remove oldest?
                     // TODO per-torrent adjustable expiration?
                     for (Iterator<Peer> iterp = p.values().iterator(); iterp.hasNext() && p.size() > MAX_PEERS_PER_TORRENT; ) {
                          iterp.next();
@@ -143,6 +159,7 @@ class DHTTracker {
                          peerCount--;
                     }
                     torrentCount++;
+                    tooMany = true;
                 } else if (recent <= 0) {
                     iter.remove();
                 } else {
@@ -151,6 +168,8 @@ class DHTTracker {
             }
 
             if (peerCount > MAX_PEERS)
+                tooMany = true;
+            if (tooMany)
                 _expireTime = Math.max(_expireTime - DELTA_EXPIRE_TIME, MIN_EXPIRE_TIME);
             else
                 _expireTime = Math.min(_expireTime + DELTA_EXPIRE_TIME, MAX_EXPIRE_TIME);
@@ -162,7 +181,7 @@ class DHTTracker {
                          DataHelper.formatDuration(_expireTime) + " expiration");
             _peerCount = peerCount;
             _torrentCount = torrentCount;
-            schedule(CLEAN_TIME);
+            schedule(tooMany ? CLEAN_TIME / 3 : CLEAN_TIME);
         }
     }
 }

apps/i2psnark/java/src/org/klomp/snark/dht/KRPC.java

@@ -38,6 +38,7 @@ import net.i2p.data.DataHelper;
 import net.i2p.data.Destination;
 import net.i2p.data.Hash;
 import net.i2p.data.SimpleDataStructure;
+import net.i2p.util.ConcurrentHashSet;
 import net.i2p.util.I2PAppThread;
 import net.i2p.util.Log;
 import net.i2p.util.SimpleTimer2;
@@ -97,6 +98,8 @@ public class KRPC implements I2PSessionMuxedListener, DHT {
     private final ConcurrentHashMap<Token, NodeInfo> _outgoingTokens;
     /** index to incoming opaque tokens, received in a peers or nodes reply */
     private final ConcurrentHashMap<NID, Token> _incomingTokens;
+    /** recently unreachable, with lastSeen() as the added-to-blacklist time  */
+    private final Set<NID> _blacklist;
 
     /** hook to inject and receive datagrams */
     private final I2PSession _session;
@@ -111,6 +114,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT {
     /** signed dgrams */
     private final int _qPort;
     private final File _dhtFile;
+    private final File _backupDhtFile;
     private volatile boolean _isRunning;
     private volatile boolean _hasBootstrapped;
     /** stats */
@@ -140,6 +144,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT {
     /** how long since generated do we delete - BEP 5 says 10 minutes */
     private static final long MAX_TOKEN_AGE = 10*60*1000;
     private static final long MAX_INBOUND_TOKEN_AGE = MAX_TOKEN_AGE - 2*60*1000;
+    private static final int MAX_OUTBOUND_TOKENS = 5000;
     /** how long since sent do we wait for a reply */
     private static final long MAX_MSGID_AGE = 2*60*1000;
     /** how long since sent do we wait for a reply */
@@ -147,12 +152,16 @@ public class KRPC implements I2PSessionMuxedListener, DHT {
     /** stagger with other cleaners */
     private static final long CLEAN_TIME = 63*1000;
     private static final long EXPLORE_TIME = 877*1000;
-    private static final String DHT_FILE = "i2psnark.dht.dat";
+    private static final long BLACKLIST_CLEAN_TIME = 17*60*1000;
+    private static final String DHT_FILE_SUFFIX = ".dht.dat";
 
     private static final int SEND_CRYPTO_TAGS = 8;
     private static final int LOW_CRYPTO_TAGS = 4;
 
-    public KRPC (I2PAppContext ctx, I2PSession session) {
+    /**
+     *  @param baseName generally "i2psnark"
+     */
+    public KRPC(I2PAppContext ctx, String baseName, I2PSession session) {
         _context = ctx;
         _session = session;
         _log = ctx.logManager().getLog(KRPC.class);
@@ -161,6 +170,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT {
         _sentQueries = new ConcurrentHashMap();
         _outgoingTokens = new ConcurrentHashMap();
         _incomingTokens = new ConcurrentHashMap();
+        _blacklist = new ConcurrentHashSet();
 
         // Construct my NodeInfo
         // Pick ports over a big range to marginally increase security
@@ -176,7 +186,8 @@ public class KRPC implements I2PSessionMuxedListener, DHT {
             _myNID = new NID(_myID);
         }
         _myNodeInfo = new NodeInfo(_myNID, session.getMyDestination(), _qPort);
-        _dhtFile = new File(ctx.getConfigDir(), DHT_FILE);
+        _dhtFile = new File(ctx.getConfigDir(), baseName + DHT_FILE_SUFFIX);
+        _backupDhtFile = baseName.equals("i2psnark") ? null : new File(ctx.getConfigDir(), "i2psnark" + DHT_FILE_SUFFIX);
         _knownNodes = new DHTNodes(ctx, _myNID);
 
         start();
@@ -262,13 +273,13 @@ public class KRPC implements I2PSessionMuxedListener, DHT {
 
             int replyType = waiter.getReplyCode();
             if (replyType == REPLY_NONE) {
-                 if (_log.shouldLog(Log.INFO))
-                     _log.info("Got no reply");
+                 if (_log.shouldLog(Log.DEBUG))
+                     _log.debug("Got no reply");
             } else if (replyType == REPLY_NODES) {
                  List<NodeInfo> reply = (List<NodeInfo>) waiter.getReplyObject();
                  // It seems like we are just going to get back ourselves all the time
-                 if (_log.shouldLog(Log.INFO))
-                     _log.info("Got " + reply.size() + " nodes");
+                 if (_log.shouldLog(Log.DEBUG))
+                     _log.debug("Got " + reply.size() + " nodes");
                  for (NodeInfo ni : reply) {
                      if (! (ni.equals(_myNodeInfo) || (toTry.contains(ni) && tried.contains(ni))))
                          toTry.add(ni);
@@ -348,14 +359,14 @@ public class KRPC implements I2PSessionMuxedListener, DHT {
 
             int replyType = waiter.getReplyCode();
             if (replyType == REPLY_NONE) {
-                 if (_log.shouldLog(Log.INFO))
-                     _log.info("Got no reply");
+                 if (_log.shouldLog(Log.DEBUG))
+                     _log.debug("Got no reply");
             } else if (replyType == REPLY_PONG) {
-                 if (_log.shouldLog(Log.INFO))
-                     _log.info("Got pong");
+                 if (_log.shouldLog(Log.DEBUG))
+                     _log.debug("Got pong");
             } else if (replyType == REPLY_PEERS) {
-                 if (_log.shouldLog(Log.INFO))
-                     _log.info("Got peers");
+                 if (_log.shouldLog(Log.DEBUG))
+                     _log.debug("Got peers");
                  List<Hash> reply = (List<Hash>) waiter.getReplyObject();
                  if (!reply.isEmpty()) {
                      for (int j = 0; j < reply.size() && rv.size() < max; j++) {
@@ -367,8 +378,8 @@ public class KRPC implements I2PSessionMuxedListener, DHT {
                  }
             } else if (replyType == REPLY_NODES) {
                  List<NodeInfo> reply = (List<NodeInfo>) waiter.getReplyObject();
-                 if (_log.shouldLog(Log.INFO))
-                     _log.info("Got " + reply.size() + " nodes");
+                 if (_log.shouldLog(Log.DEBUG))
+                     _log.debug("Got " + reply.size() + " nodes");
                  for (NodeInfo ni : reply) {
                      if (! (ni.equals(_myNodeInfo) || tried.contains(ni) || toTry.contains(ni)))
                          toTry.add(ni);
@@ -541,7 +552,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT {
         _session.addMuxedSessionListener(this, I2PSession.PROTO_DATAGRAM, _qPort);
         _knownNodes.start();
         _tracker.start();
-        PersistDHT.loadDHT(this, _dhtFile);
+        PersistDHT.loadDHT(this, _dhtFile, _backupDhtFile);
         // start the explore thread
         _isRunning = true;
         // no need to keep ref, it will eventually stop
@@ -567,7 +578,9 @@ public class KRPC implements I2PSessionMuxedListener, DHT {
         _session.removeListener(I2PSession.PROTO_DATAGRAM_RAW, _rPort);
         // clear the DHT and tracker
         _tracker.stop();
-        PersistDHT.saveDHT(_knownNodes, _dhtFile);
+        // don't lose all our peers if we didn't have time to check them
+        boolean saveAll = _context.clock().now() - _started < 20*60*1000;
+        PersistDHT.saveDHT(_knownNodes, saveAll, _dhtFile);
         _knownNodes.stop();
         for (Iterator<ReplyWaiter> iter = _sentQueries.values().iterator(); iter.hasNext(); ) {
             ReplyWaiter waiter = iter.next();
@@ -576,6 +589,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT {
         }
         _outgoingTokens.clear();
         _incomingTokens.clear();
+        _blacklist.clear();
     }
 
     /**
@@ -592,7 +606,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT {
      */
     public String renderStatusHTML() {
         long uptime = Math.max(1000, _context.clock().now() - _started);
-        StringBuilder buf = new StringBuilder();
+        StringBuilder buf = new StringBuilder(256);
         buf.append("<br><b>DHT DEBUG</b><br>TX: ").append(_txPkts.get()).append(" pkts / ")
            .append(DataHelper.formatSize2(_txBytes.get())).append("B / ")
            .append(DataHelper.formatSize2(_txBytes.get() * 1000 / uptime)).append("Bps<br>" +
@@ -600,10 +614,12 @@ public class KRPC implements I2PSessionMuxedListener, DHT {
            .append(DataHelper.formatSize2(_rxBytes.get())).append("B / ")
            .append(DataHelper.formatSize2(_rxBytes.get() * 1000 / uptime)).append("Bps<br>" +
                    "DHT Peers: ").append( _knownNodes.size()).append("<br>" +
+                   "Blacklisted: ").append(_blacklist.size()).append("<br>" +
                    "Sent tokens: ").append(_outgoingTokens.size()).append("<br>" +
                    "Rcvd tokens: ").append(_incomingTokens.size()).append("<br>" +
                    "Pending queries: ").append(_sentQueries.size()).append("<br>");
         _tracker.renderStatusHTML(buf);
+        _knownNodes.renderStatusHTML(buf);
         return buf.toString();
     }
 
@@ -1079,7 +1095,12 @@ public class KRPC implements I2PSessionMuxedListener, DHT {
             if (oldInfo.getDestination() == null && nInfo.getDestination() != null)
                 oldInfo.setDestination(nInfo.getDestination());
         }
-        oldInfo.getNID().setLastSeen();
+        nID = oldInfo.getNID();
+        nID.setLastSeen();
+        if (_blacklist.remove(nID)) {
+            if (_log.shouldLog(Log.INFO))
+                _log.info("UN-blacklisted: " + nID);
+        }
         return oldInfo;
     }
 
@@ -1093,8 +1114,12 @@ public class KRPC implements I2PSessionMuxedListener, DHT {
         if (nInfo.equals(_myNodeInfo))
             return _myNodeInfo;
         NodeInfo rv = _knownNodes.putIfAbsent(nInfo);
-        if (rv == null)
+        if (rv == null) {
             rv = nInfo;
+            // if we didn't know about it before, set the timestamp
+            // so it isn't immediately removed by the DHTNodes cleaner
+            rv.getNID().setLastSeen();
+        }
         return rv;
     }
 
@@ -1109,6 +1134,13 @@ public class KRPC implements I2PSessionMuxedListener, DHT {
                 if (_log.shouldLog(Log.INFO))
                     _log.info("Removed after consecutive timeouts: " + nInfo);
             }
+            if (!_blacklist.contains(nid)) {
+                // used as when-added time
+                nid.setLastSeen();
+                _blacklist.add(nid);
+                if (_log.shouldLog(Log.INFO))
+                    _log.info("Blacklisted: " + nid);
+            }
         }
     }
 
@@ -1182,7 +1214,8 @@ public class KRPC implements I2PSessionMuxedListener, DHT {
 
     /**
      *  Handle and respond to the query.
-     *  We have no node info here, it came on response port, we have to get it from the token
+     *  We have no node info here, it came on response port, we have to get it from the token.
+     *  So we can't verify that it came from the same peer, as BEP 5 specifies.
      */
     private void receiveAnnouncePeer(MsgID msgID, InfoHash ih, byte[] tok) throws InvalidBEncodingException {
         Token token = new Token(tok);
@@ -1190,8 +1223,8 @@ public class KRPC implements I2PSessionMuxedListener, DHT {
         if (nInfo == null) {
             if (_log.shouldLog(Log.WARN))
                 _log.warn("Unknown token in announce_peer: " + token);
-            if (_log.shouldLog(Log.INFO))
-                _log.info("Current known tokens: " + _outgoingTokens.keySet());
+            //if (_log.shouldLog(Log.INFO))
+            //    _log.info("Current known tokens: " + _outgoingTokens.keySet());
             return;
         }
         if (_log.shouldLog(Log.INFO))
@@ -1223,11 +1256,11 @@ public class KRPC implements I2PSessionMuxedListener, DHT {
                 byte[] tok = btok.getBytes();
                 Token token = new Token(_context, tok);
                 _incomingTokens.put(nInfo.getNID(), token);
-                if (_log.shouldLog(Log.INFO))
-                    _log.info("Got token: " + token + ", must be a response to get_peers");
+                if (_log.shouldLog(Log.DEBUG))
+                    _log.debug("Got token: " + token + ", must be a response to get_peers");
             } else {
-                if (_log.shouldLog(Log.INFO))
-                    _log.info("No token and saved infohash, must be a response to find_node");
+                if (_log.shouldLog(Log.DEBUG))
+                    _log.debug("No token and saved infohash, must be a response to find_node");
             }
         }
 
@@ -1256,9 +1289,15 @@ public class KRPC implements I2PSessionMuxedListener, DHT {
      *  @throws NPE, IllegalArgumentException, and others too
      */
     private List<NodeInfo> receiveNodes(NodeInfo nInfo, byte[] ids) throws InvalidBEncodingException {
-        List<NodeInfo> rv = new ArrayList(ids.length / NodeInfo.LENGTH);
-        for (int off = 0; off < ids.length; off += NodeInfo.LENGTH) {
+        int max = Math.min(K, ids.length / NodeInfo.LENGTH);
+        List<NodeInfo> rv = new ArrayList(max);
+        for (int off = 0; off < ids.length && rv.size() < max; off += NodeInfo.LENGTH) {
             NodeInfo nInf = new NodeInfo(ids, off);
+            if (_blacklist.contains(nInf.getNID())) {
+                if (_log.shouldLog(Log.INFO))
+                    _log.info("Ignoring blacklisted " + nInf.getNID() + " from: " + nInfo);
+                continue;
+            }
             nInf = heardAbout(nInf);
             rv.add(nInf);
         }
@@ -1274,12 +1313,15 @@ public class KRPC implements I2PSessionMuxedListener, DHT {
     private List<Hash> receivePeers(NodeInfo nInfo, List<BEValue> peers) throws InvalidBEncodingException {
         if (_log.shouldLog(Log.INFO))
              _log.info("Rcvd peers from: " + nInfo);
-        List<Hash> rv = new ArrayList(peers.size());
+        int max = Math.min(MAX_WANT, peers.size());
+        List<Hash> rv = new ArrayList(max);
         for (BEValue bev : peers) {
             byte[] b = bev.getBytes();
             //Hash h = new Hash(b);
             Hash h = Hash.create(b);
             rv.add(h);
+            if (rv.size() >= max)
+                break;
         }
         if (_log.shouldLog(Log.INFO))
              _log.info("Rcvd peers from: " + nInfo + ": " + DataHelper.toString(rv));
@@ -1492,7 +1534,7 @@ public class KRPC implements I2PSessionMuxedListener, DHT {
     private class Cleaner extends SimpleTimer2.TimedEvent {
 
         public Cleaner() {
-            super(SimpleTimer2.getInstance(), CLEAN_TIME);
+            super(SimpleTimer2.getInstance(), 7 * CLEAN_TIME);
         }
 
         public void timeReached() {
@@ -1501,21 +1543,37 @@ public class KRPC implements I2PSessionMuxedListener, DHT {
             long now = _context.clock().now();
             if (_log.shouldLog(Log.DEBUG))
                 _log.debug("KRPC cleaner starting with " +
+                          _blacklist.size() + " in blacklist, " +
                           _outgoingTokens.size() + " sent Tokens, " +
                           _incomingTokens.size() + " rcvd Tokens");
+            int cnt = 0;
+            long expire = now - MAX_TOKEN_AGE;
             for (Iterator<Token> iter = _outgoingTokens.keySet().iterator(); iter.hasNext(); ) {
                 Token tok = iter.next();
-                if (tok.lastSeen() < now - MAX_TOKEN_AGE)
+                // just delete at random if we have too many
+                // TODO reduce the expire time and iterate again?
+                if (tok.lastSeen() < expire || cnt >= MAX_OUTBOUND_TOKENS)
                     iter.remove();
+                else
+                    cnt++;
             }
+            expire = now - MAX_INBOUND_TOKEN_AGE;
             for (Iterator<Token> iter = _incomingTokens.values().iterator(); iter.hasNext(); ) {
                 Token tok = iter.next();
-                if (tok.lastSeen() < now - MAX_INBOUND_TOKEN_AGE)
+                if (tok.lastSeen() < expire)
+                    iter.remove();
+            }
+            expire = now - BLACKLIST_CLEAN_TIME;
+            for (Iterator<NID> iter = _blacklist.iterator(); iter.hasNext(); ) {
+                NID nid = iter.next();
+                // lastSeen() is actually when-added
+                if (nid.lastSeen() < expire)
                     iter.remove();
             }
             // TODO sent queries?
             if (_log.shouldLog(Log.DEBUG))
                 _log.debug("KRPC cleaner done, now with " +
+                          _blacklist.size() + " in blacklist, " +
                           _outgoingTokens.size() + " sent Tokens, " +
                           _incomingTokens.size() + " rcvd Tokens, " +
                           _knownNodes.size() + " known peers, " +

apps/i2psnark/java/src/org/klomp/snark/dht/NID.java

@@ -18,7 +18,7 @@ public class NID extends SHA1Hash {
     private long lastSeen;
     private int fails;
 
-    private static final int MAX_FAILS = 3;
+    private static final int MAX_FAILS = 2;
 
     public NID() {
         super(null);
@@ -41,6 +41,6 @@ public class NID extends SHA1Hash {
      *  @return if more than max timeouts
      */
     public boolean timeout() {
-        return fails++ > MAX_FAILS;
+        return ++fails > MAX_FAILS;
     }
 }

apps/i2psnark/java/src/org/klomp/snark/dht/PersistDHT.java

@@ -23,6 +23,17 @@ abstract class PersistDHT {
 
     private static final long MAX_AGE = 60*60*1000;
 
+    /**
+     *  @param backupFile may be null
+     *  @since 0.9.6
+     */
+    public static synchronized void loadDHT(KRPC krpc, File file, File backupFile) {
+        if (file.exists())
+            loadDHT(krpc, file);
+        else if (backupFile != null)
+            loadDHT(krpc, backupFile);
+    }
+
     public static synchronized void loadDHT(KRPC krpc, File file) {
         Log log = I2PAppContext.getGlobalContext().logManager().getLog(PersistDHT.class);
         int count = 0;
@@ -56,12 +67,15 @@ abstract class PersistDHT {
             log.info("Loaded " + count + " nodes from " + file);
     }
 
-    public static synchronized void saveDHT(DHTNodes nodes, File file) {
+    /**
+     *  @param saveAll if true, don't check last seen time
+     */
+    public static synchronized void saveDHT(DHTNodes nodes, boolean saveAll, File file) {
         if (nodes.size() <= 0)
             return;
         Log log = I2PAppContext.getGlobalContext().logManager().getLog(PersistDHT.class);
         int count = 0;
-        long maxAge = I2PAppContext.getGlobalContext().clock().now() - MAX_AGE;
+        long maxAge = saveAll ? 0 : I2PAppContext.getGlobalContext().clock().now() - MAX_AGE;
         PrintWriter out = null;
         try {
             out = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new SecureFileOutputStream(file), "ISO-8859-1")));

apps/i2psnark/java/src/org/klomp/snark/web/BasicServlet.java

@@ -0,0 +1,599 @@
+// ========================================================================
+// Copyright 199-2004 Mort Bay Consulting Pty. Ltd.
+// ------------------------------------------------------------------------
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at 
+// http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ========================================================================
+
+package org.klomp.snark.web;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Enumeration;
+import java.util.List;
+
+import javax.servlet.ServletConfig;
+import javax.servlet.ServletContext;
+import javax.servlet.ServletException;
+import javax.servlet.UnavailableException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import net.i2p.I2PAppContext;
+import net.i2p.data.ByteArray;
+import net.i2p.util.ByteCache;
+import net.i2p.util.Log;
+
+
+/* ------------------------------------------------------------ */
+/**
+ * Based on DefaultServlet from Jetty 6.1.26, heavily simplified
+ * and modified to remove all dependencies on Jetty libs.
+ *
+ * Supports HEAD and GET only, for resources from the .war and local files.
+ * Supports files and resource only.
+ * Supports MIME types with local overrides and additions.
+ * Supports Last-Modified.
+ * Supports single request ranges.
+ *
+ * Does not support directories or "welcome files".
+ * Does not support gzip.
+ * Does not support multiple request ranges.
+ * Does not cache.
+ *
+ * POST returns 405.
+ * Directories return 403.
+ * Jar resources are sent with a long cache directive.
+ *
+ * ------------------------------------------------------------ 
+ *
+ * The default servlet.                                                 
+ * This servlet, normally mapped to /, provides the handling for static 
+ * content, OPTION and TRACE methods for the context.                   
+ * The following initParameters are supported, these can be set either
+ * on the servlet itself or as ServletContext initParameters with a prefix
+ * of org.mortbay.jetty.servlet.Default. :                          
+ * <PRE>                                                                      
+ *
+ *  resourceBase      Set to replace the context resource base
+
+ *  warBase      Path allowed for resource in war
+ * 
+ * </PRE>
+ *                                                                    
+ *
+ * @author Greg Wilkins (gregw)
+ * @author Nigel Canonizado
+ *                                                                    
+ * @since Jetty 7
+ */
+class BasicServlet extends HttpServlet
+{   
+    protected final I2PAppContext _context;
+    protected final Log _log;
+    protected File _resourceBase;
+    private String _warBase;
+    
+    private final MimeTypes _mimeTypes;
+    
+    /** same as PeerState.PARTSIZE */
+    private static final int BUFSIZE = 16*1024;
+    private ByteCache _cache = ByteCache.getInstance(16, BUFSIZE);
+
+    private static final int WAR_CACHE_CONTROL_SECS = 24*60*60;
+    private static final int FILE_CACHE_CONTROL_SECS = 24*60*60;
+
+    public BasicServlet() {
+        super();
+        _context = I2PAppContext.getGlobalContext();
+        _log = _context.logManager().getLog(getClass());
+        _mimeTypes = new MimeTypes();
+    }
+    
+    /* ------------------------------------------------------------ */
+    public void init(ServletConfig cfg) throws ServletException {
+        super.init(cfg);
+        String rb=getInitParameter("resourceBase");
+        if (rb!=null)
+        {
+            File f = new File(rb);
+            setResourceBase(f);
+        }
+        String wb = getInitParameter("warBase");
+        if (wb != null)
+            setWarBase(wb);
+    }
+
+    /**
+     *  Files are served from here
+     */
+    protected void setResourceBase(File base) throws UnavailableException {
+        if (!base.isDirectory())
+            throw new UnavailableException("Resource base does not exist: " + base);
+        _resourceBase = base;
+        if (_log.shouldLog(Log.INFO))
+            _log.info("Resource base is " + _resourceBase);
+    }
+
+    /**
+     *  Only paths starting with this in the path are served
+     */
+    protected void setWarBase(String base) {
+        if (!base.startsWith("/"))
+            base = '/' + base;
+        if (!base.endsWith("/"))
+            base = base + '/';
+        _warBase = base;
+        if (_log.shouldLog(Log.INFO))
+            _log.info("War base is " + _warBase);
+    }
+
+    /** get Resource to serve.
+     * Map a path to a resource. The default implementation calls
+     * HttpContext.getResource but derived servlets may provide
+     * their own mapping.
+     * @param pathInContext The path to find a resource for.
+     * @return The resource to serve or null if not existing
+     */
+    public File getResource(String pathInContext)
+    {
+        if (_resourceBase==null)
+            return null;
+        File r = null;
+        if (!pathInContext.contains("..") &&
+                   !pathInContext.endsWith("/")) {
+            File f = new File(_resourceBase, pathInContext);
+            if (f.exists())
+                r = f;
+        }
+        return r;
+    }
+
+    /** get Resource to serve.
+     * Map a path to a resource. The default implementation calls
+     * HttpContext.getResource but derived servlets may provide
+     * their own mapping.
+     * @param pathInContext The path to find a resource for.
+     * @return The resource to serve or null. Returns null for directories
+     */
+    public HttpContent getContent(String pathInContext)
+    {
+        if (_resourceBase==null)
+            return null;
+        HttpContent r = null;
+        if (_warBase != null && pathInContext.startsWith(_warBase)) {
+            r = new JarContent(pathInContext);
+        } else if (!pathInContext.contains("..") &&
+                   !pathInContext.endsWith("/")) {
+            File f = new File(_resourceBase, pathInContext);
+            // exists && !directory
+            if (f.isFile())
+                r = new FileContent(f);
+        }
+        return r;
+    }
+    
+    /* ------------------------------------------------------------ */
+    protected void doGet(HttpServletRequest request, HttpServletResponse response)
+    	throws ServletException, IOException
+    {
+        // always starts with a '/'
+        String servletpath = request.getServletPath();
+        String pathInfo=request.getPathInfo();
+        // ??? right??
+        String pathInContext = addPaths(servletpath, pathInfo);        
+
+        // Find the resource and content
+        try {
+            HttpContent content = getContent(pathInContext);
+            
+            // Handle resource
+            if (content == null) {
+                if (_log.shouldLog(Log.WARN))
+                    _log.warn("Not found: " + pathInContext);
+                response.sendError(404);
+            } else {
+                if (passConditionalHeaders(request, response, content)) {
+                    if (_log.shouldLog(Log.INFO))
+                        _log.info("Sending: " + content);
+                    sendData(request, response, content);  
+                } else {
+                    if (_log.shouldLog(Log.INFO))
+                        _log.info("Not modified: " + content);
+                }
+            }
+        }
+        catch(IllegalArgumentException e)
+        {
+            if (_log.shouldLog(Log.WARN))
+                _log.warn("Error sending " + pathInContext, e);
+            if(!response.isCommitted())
+                response.sendError(500, e.getMessage());
+        }
+        catch(IOException e)
+        {
+            if (_log.shouldLog(Log.WARN))
+                // typical browser abort
+                //_log.warn("Error sending", e);
+                _log.warn("Error sending " + pathInContext + ": " + e);
+            throw e;
+        }
+    }
+    
+    /* ------------------------------------------------------------ */
+    protected void doPost(HttpServletRequest request, HttpServletResponse response)
+        throws ServletException, IOException
+    {
+        response.sendError(405);
+    }
+    
+    /* ------------------------------------------------------------ */
+    /* (non-Javadoc)
+     * @see javax.servlet.http.HttpServlet#doTrace(javax.servlet.http.HttpServletRequest, javax.servlet.http.HttpServletResponse)
+     */
+    protected void doTrace(HttpServletRequest request, HttpServletResponse response)
+        throws ServletException, IOException
+    {
+        response.sendError(405);
+    }
+
+    protected void doOptions(HttpServletRequest request, HttpServletResponse response)
+        throws ServletException, IOException
+    {
+        response.sendError(405);
+    }
+    
+    protected void doDelete(HttpServletRequest request, HttpServletResponse response)
+        throws ServletException, IOException
+    {
+        response.sendError(405);
+    }
+    
+
+    /* ------------------------------------------------------------ */
+    /** Check modification date headers.
+     *  @return true to keep going, false if handled here
+     */
+    protected boolean passConditionalHeaders(HttpServletRequest request,HttpServletResponse response, HttpContent content)
+    throws IOException
+    {
+        try
+        {
+            if (!request.getMethod().equals("HEAD") ) {
+                long ifmsl=request.getDateHeader("If-Modified-Since");
+                if (ifmsl!=-1)
+                {
+                    if (content.getLastModified()/1000 <= ifmsl/1000)
+                    {
+                        response.reset();
+                        response.setStatus(304);
+                        response.flushBuffer();
+                        return false;
+                    }
+                }
+            }
+        }
+        catch(IllegalArgumentException iae)
+        {
+            if(!response.isCommitted())
+                response.sendError(400, iae.getMessage());
+            throw iae;
+        }
+        return true;
+    }
+    
+    
+    /* ------------------------------------------------------------ */
+    protected void sendData(HttpServletRequest request,
+                            HttpServletResponse response,
+                            HttpContent content)
+    throws IOException
+    {
+        InputStream in =null;
+        try {
+            in = content.getInputStream();
+        } catch (IOException e) {
+            if (_log.shouldLog(Log.WARN))
+                _log.warn("Not found: " + content);
+            response.sendError(404);
+            return;
+        }
+
+        OutputStream out =null;
+        try {
+            out = response.getOutputStream();
+        } catch (IllegalStateException e) {
+            out = new WriterOutputStream(response.getWriter());
+        }
+
+        long content_length = content.getContentLength();
+
+        // see if there are any range headers
+        Enumeration reqRanges = request.getHeaders("Range");
+
+        if (reqRanges == null || !reqRanges.hasMoreElements()) {
+            // if there were no ranges, send entire entity
+            // Write content normally
+            writeHeaders(response,content,content_length);
+            if (content_length >= 0  && request.getMethod().equals("HEAD")) {
+                // if we know the content length, don't send it to be counted
+                if (_log.shouldLog(Log.INFO))
+                    _log.info("HEAD: " + content);
+            } else {
+                // GET or unknown size for HEAD
+                copy(in, out);
+            }
+            return;
+        }
+
+
+        // Parse the satisfiable ranges
+        List<InclusiveByteRange> ranges = InclusiveByteRange.satisfiableRanges(reqRanges, content_length);
+
+        // if there are no satisfiable ranges, send 416 response
+        // Completely punt on multiple ranges (unlike Default)
+        if (ranges == null || ranges.size() != 1) {
+            writeHeaders(response, content, content_length);
+            response.setStatus(416);
+            response.setHeader("Content-Range", InclusiveByteRange.to416HeaderRangeString(content_length));
+            return;
+        }
+
+        // if there is only a single valid range (must be satisfiable
+        // since were here now), send that range with a 216 response
+        InclusiveByteRange singleSatisfiableRange = ranges.get(0);
+        long singleLength = singleSatisfiableRange.getSize(content_length);
+        writeHeaders(response, content, singleLength);
+        response.setStatus(206);
+        response.setHeader("Content-Range", singleSatisfiableRange.toHeaderRangeString(content_length));
+        copy(in, singleSatisfiableRange.getFirst(content_length), out, singleLength);
+    }
+    
+    /* ------------------------------------------------------------ */
+    protected void writeHeaders(HttpServletResponse response,HttpContent content,long count)
+        throws IOException
+    {   
+        if (content.getContentType()!=null && response.getContentType()==null)
+            response.setContentType(content.getContentType());
+        
+        long lml = content.getLastModified();
+        if (lml > 0)
+            response.setDateHeader("Last-Modified",lml);
+
+        if (count != -1)
+        {
+            if (count<Integer.MAX_VALUE)
+                response.setContentLength((int)count);
+            else 
+                response.setHeader("Content-Length", Long.toString(count));
+        }
+
+        long ct = content.getCacheTime();
+        if (ct>=0)
+            response.setHeader("Cache-Control", "public, max-age=" + ct);
+
+    }
+
+    /* ------------------------------------------------------------ */
+    /* ------------------------------------------------------------ */
+    /* ------------------------------------------------------------ */
+    /* I2P additions below here */
+
+    /** from Jetty HttpContent.java */
+    public interface HttpContent
+    {
+        String getContentType();
+        long getLastModified();
+        /** in seconds */
+        int getCacheTime();
+        long getContentLength();
+        InputStream getInputStream() throws IOException;
+    }
+
+    private class FileContent implements HttpContent
+    {
+        private final File _file;
+        
+        public FileContent(File file)
+        {
+            _file = file;
+        }
+        
+        /* ------------------------------------------------------------ */
+        public String getContentType()
+        {
+            //return _mimeTypes.getMimeByExtension(_file.toString());
+            return getMimeType(_file.toString());
+        }
+
+        /* ------------------------------------------------------------ */
+        public long getLastModified()
+        {
+            return _file.lastModified();
+        }
+
+        public int getCacheTime()
+        {
+            return FILE_CACHE_CONTROL_SECS;
+        }
+
+        /* ------------------------------------------------------------ */
+        public long getContentLength()
+        {
+            return _file.length();
+        }
+
+        /* ------------------------------------------------------------ */
+        public InputStream getInputStream() throws IOException
+        {
+            return new BufferedInputStream(new FileInputStream(_file));
+        }
+
+        @Override
+        public String toString() { return "File \"" + _file + '"'; }
+    }
+
+    private class JarContent implements HttpContent
+    {
+        private final String _path;
+        
+        public JarContent(String path)
+        {
+            _path = path;
+        }
+        
+        /* ------------------------------------------------------------ */
+        public String getContentType()
+        {
+            return getMimeType(_path);
+        }
+
+        /* ------------------------------------------------------------ */
+        public long getLastModified()
+        {
+            String cpath = getServletContext().getContextPath();
+            // this won't work if we aren't at top level
+            String cname = cpath == "" ? "i2psnark" : cpath.substring(1).replace("/", "_");
+            return (new File(_context.getBaseDir(), "webapps/" + cname + ".war")).lastModified();
+        }
+
+        public int getCacheTime()
+        {
+            return WAR_CACHE_CONTROL_SECS;
+        }
+
+        /* ------------------------------------------------------------ */
+        public long getContentLength()
+        {
+            return -1;
+        }
+
+        /* ------------------------------------------------------------ */
+        public InputStream getInputStream() throws IOException
+        {
+            InputStream rv = getServletContext().getResourceAsStream(_path);
+            if (rv == null)
+                throw new IOException("Not found");
+            return rv;
+        }
+
+        @Override
+        public String toString() { return "Jar resource \"" + _path + '"'; }
+    }
+
+
+    /**
+     * @param resourcePath in the classpath, without ".properties" extension
+     */
+    protected void loadMimeMap(String resourcePath) {
+        _mimeTypes.loadMimeMap(resourcePath);
+    }
+
+    /* ------------------------------------------------------------ */
+    /** Get the MIME type by filename extension.
+     * @param filename A file name
+     * @return MIME type matching the longest dot extension of the
+     * file name.
+     */
+    protected String getMimeType(String filename) {
+        String rv = _mimeTypes.getMimeByExtension(filename);
+        if (rv != null)
+            return rv;
+        return getServletContext().getMimeType(filename);
+    }
+
+    protected void addMimeMapping(String extension, String type) {
+        _mimeTypes.addMimeMapping(extension, type);
+    }
+
+    /**
+     *  Simple version of URIUtil.addPaths()
+     *  @param path may be null
+     */
+    protected static String addPaths(String base, String path) {
+        if (path == null)
+            return base;
+        return (new File(base, path)).toString();
+    }
+
+    /**
+     *  Simple version of URIUtil.decodePath()
+     */
+    protected static String decodePath(String path) throws MalformedURLException {
+        if (!path.contains("%"))
+            return path;
+        try {
+            URI uri = new URI(path);
+            return uri.getPath();
+        } catch (URISyntaxException use) {
+            // for ease of use, since a USE is not an IOE but a MUE is...
+            throw new MalformedURLException(use.getMessage());
+        }
+    }
+
+    /**
+     *  Simple version of URIUtil.encodePath()
+     */
+    protected static String encodePath(String path) throws MalformedURLException {
+        try {
+            URI uri = new URI(null, null, path, null);
+            return uri.toString();
+        } catch (URISyntaxException use) {
+            // for ease of use, since a USE is not an IOE but a MUE is...
+            throw new MalformedURLException(use.getMessage());
+        }
+    }
+
+    /**
+     *  Write from in to out
+     */
+    private void copy(InputStream in, OutputStream out) throws IOException {
+        copy(in, 0, out, -1);
+    }
+
+    /**
+     *  Write from in to out
+     */
+    private void copy(InputStream in, long skip, OutputStream out, final long len) throws IOException {
+        ByteArray ba = _cache.acquire();
+        byte[] buf = ba.getData();
+        try {
+            if (skip > 0)
+                in.skip(skip);
+            int read = 0;
+            long tot = 0;
+            boolean done = false;
+            while ( (read = in.read(buf)) != -1 && !done) {
+                if (len >= 0) {
+                    tot += read;
+                    if (tot >= len) {
+                        read -= (int) (tot - len);
+                        done = true;
+                    }
+                }
+                out.write(buf, 0, read);
+            }
+        } finally {
+            _cache.release(ba, false);
+            if (in != null) 
+                try { in.close(); } catch (IOException ioe) {}
+            if (out != null) 
+                try { out.close(); } catch (IOException ioe) {}
+        }
+    }
+}

apps/i2psnark/java/src/org/klomp/snark/web/InclusiveByteRange.java

@@ -0,0 +1,218 @@
+//
+//  ========================================================================
+//  Copyright (c) 1995-2013 Mort Bay Consulting Pty. Ltd.
+//  ------------------------------------------------------------------------
+//  All rights reserved. This program and the accompanying materials
+//  are made available under the terms of the Eclipse Public License v1.0
+//  and Apache License v2.0 which accompanies this distribution.
+//
+//      The Eclipse Public License is available at
+//      http://www.eclipse.org/legal/epl-v10.html
+//
+//      The Apache License v2.0 is available at
+//      http://www.opensource.org/licenses/apache2.0.php
+//
+//  You may elect to redistribute this code under either of these licenses.
+//  ========================================================================
+//
+
+package org.klomp.snark.web;
+
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.StringTokenizer;
+
+/* ------------------------------------------------------------ */
+/** Byte range inclusive of end points.
+ * <PRE>
+ * 
+ *   parses the following types of byte ranges:
+ * 
+ *       bytes=100-499
+ *       bytes=-300
+ *       bytes=100-
+ *       bytes=1-2,2-3,6-,-2
+ *
+ *   given an entity length, converts range to string
+ * 
+ *       bytes 100-499/500
+ * 
+ * </PRE>
+ * 
+ * Based on RFC2616 3.12, 14.16, 14.35.1, 14.35.2
+ * @version $version$
+ * 
+ */
+public class InclusiveByteRange 
+{
+    long first = 0;
+    long last  = 0;    
+
+    public InclusiveByteRange(long first, long last)
+    {
+        this.first = first;
+        this.last = last;
+    }
+    
+    public long getFirst()
+    {
+        return first;
+    }
+
+    public long getLast()
+    {
+        return last;
+    }    
+
+
+    
+    /* ------------------------------------------------------------ */
+    /** 
+     * @param headers Enumeration of Range header fields.
+     * @param size Size of the resource.
+     * @return List of satisfiable ranges
+     */
+    public static List<InclusiveByteRange> satisfiableRanges(Enumeration headers, long size)
+    {
+        List<InclusiveByteRange> satRanges = null;
+        
+        // walk through all Range headers
+    headers:
+        while (headers.hasMoreElements())
+        {
+            String header = (String) headers.nextElement();
+            StringTokenizer tok = new StringTokenizer(header,"=,",false);
+            String t=null;
+            try
+            {
+                // read all byte ranges for this header 
+                while (tok.hasMoreTokens())
+                {
+                    try
+                    {
+                        t = tok.nextToken().trim();
+
+                        long first = -1;
+                        long last = -1;
+                        int d = t.indexOf('-');
+                        if (d < 0 || t.indexOf("-",d + 1) >= 0)
+                        {
+                            if ("bytes".equals(t))
+                                continue;
+                            continue headers;
+                        }
+                        else if (d == 0)
+                        {
+                            if (d + 1 < t.length())
+                                last = Long.parseLong(t.substring(d + 1).trim());
+                            else
+                            {
+                                continue;
+                            }
+                        }
+                        else if (d + 1 < t.length())
+                        {
+                            first = Long.parseLong(t.substring(0,d).trim());
+                            last = Long.parseLong(t.substring(d + 1).trim());
+                        }
+                        else
+                            first = Long.parseLong(t.substring(0,d).trim());
+
+                        if (first == -1 && last == -1)
+                            continue headers;
+
+                        if (first != -1 && last != -1 && (first > last))
+                            continue headers;
+
+                        if (first < size)
+                        {
+                            if (satRanges == null)
+                                satRanges = new ArrayList(4);
+                            InclusiveByteRange range = new InclusiveByteRange(first,last);
+                            satRanges.add(range);
+                        }
+                    }
+                    catch (NumberFormatException e)
+                    {
+                        continue;
+                    }
+                }
+            }
+            catch(Exception e)
+            {
+            }    
+        }
+        return satRanges;
+    }
+
+    /* ------------------------------------------------------------ */
+    public long getFirst(long size)
+    {
+        if (first<0)
+        {
+            long tf=size-last;
+            if (tf<0)
+                tf=0;
+            return tf;
+        }
+        return first;
+    }
+    
+    /* ------------------------------------------------------------ */
+    public long getLast(long size)
+    {
+        if (first<0)
+            return size-1;
+        
+        if (last<0 ||last>=size)
+            return size-1;
+        return last;
+    }
+    
+    /* ------------------------------------------------------------ */
+    public long getSize(long size)
+    {
+        return getLast(size)-getFirst(size)+1;
+    }
+
+
+    /* ------------------------------------------------------------ */
+    public String toHeaderRangeString(long size)
+    {
+        StringBuilder sb = new StringBuilder(40);
+        sb.append("bytes ");
+        sb.append(getFirst(size));
+        sb.append('-');
+        sb.append(getLast(size));
+        sb.append("/");
+        sb.append(size);
+        return sb.toString();
+    }
+
+    /* ------------------------------------------------------------ */
+    public static String to416HeaderRangeString(long size)
+    {
+        StringBuilder sb = new StringBuilder(40);
+        sb.append("bytes */");
+        sb.append(size);
+        return sb.toString();
+    }
+
+
+    /* ------------------------------------------------------------ */
+    @Override
+    public String toString()
+    {
+        StringBuilder sb = new StringBuilder(60);
+        sb.append(Long.toString(first));
+        sb.append(":");
+        sb.append(Long.toString(last));
+        return sb.toString();
+    }
+
+
+}
+
+
+

apps/i2psnark/java/src/org/klomp/snark/web/MimeTypes.java

@@ -0,0 +1,131 @@
+// ========================================================================
+// Copyright 2000-2005 Mort Bay Consulting Pty. Ltd.
+// ------------------------------------------------------------------------
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at 
+// http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ========================================================================
+
+package org.klomp.snark.web;
+
+import java.util.Enumeration;
+import java.util.Locale;
+import java.util.Map;
+import java.util.MissingResourceException;
+import java.util.ResourceBundle;
+import java.util.concurrent.ConcurrentHashMap;
+
+import javax.servlet.ServletContext;
+
+
+/* ------------------------------------------------------------ */
+/**
+ * Based on MimeTypes from Jetty 6.1.26, heavily simplified
+ * and modified to remove all dependencies on Jetty libs.
+ *
+ * Supports mime types only, not encodings.
+ * Does not support a default "*" mapping.
+ *
+ *  This is only for local mappings.
+ *  Caller should use getServletContext().getMimeType() if this returns null.
+ *
+ *
+ * ------------------------------------------------------------ 
+ *
+ * @author Greg Wilkins
+ *
+ * @since Jetty 7
+ */
+class MimeTypes
+{
+    
+    private final Map<String, String> _mimeMap;
+
+    public MimeTypes() {
+        _mimeMap = new ConcurrentHashMap();
+    }
+
+    /* ------------------------------------------------------------ */
+    /**
+     * @param resourcePath A Map of file extension to mime-type.
+     */
+    public void loadMimeMap(String resourcePath) {
+        loadMimeMap(_mimeMap, resourcePath);
+    }
+
+    /**
+     *  Tries both webapp and system class loader, since Jetty blocks
+     *  its classes from the webapp class loader.
+     */
+    private static void loadMimeMap(Map<String, String> map, String resourcePath) {
+        try
+        {
+            ResourceBundle mime;
+            try {
+                mime = ResourceBundle.getBundle(resourcePath);
+            } catch(MissingResourceException e) {
+                // Jetty 7 webapp classloader blocks jetty classes
+                // http://wiki.eclipse.org/Jetty/Reference/Jetty_Classloading
+                //System.out.println("No mime types loaded from " + resourcePath + ", trying system classloader");
+                mime = ResourceBundle.getBundle(resourcePath, Locale.getDefault(), ClassLoader.getSystemClassLoader());
+            }
+            Enumeration<String> i = mime.getKeys();
+            while(i.hasMoreElements())
+            {
+                String ext = i.nextElement();
+                String m = mime.getString(ext);
+                map.put(ext.toLowerCase(Locale.US), m);
+            }
+            //System.out.println("Loaded " + map.size() + " mime types from " + resourcePath);
+        } catch(MissingResourceException e) {
+            //System.out.println("No mime types loaded from " + resourcePath);
+        }
+    }
+
+    /* ------------------------------------------------------------ */
+    /** Get the MIME type by filename extension.
+     *
+     *  Returns ONLY local mappings.
+     *  Caller should use getServletContext().getMimeType() if this returns null.
+     *
+     * @param filename A file name
+     * @return MIME type matching the longest dot extension of the
+     * file name.
+     */
+    public String getMimeByExtension(String filename)
+    {
+        String type=null;
+
+        if (filename!=null)
+        {
+            int i=-1;
+            while(type==null)
+            {
+                i=filename.indexOf(".",i+1);
+
+                if (i<0 || i>=filename.length())
+                    break;
+
+                String ext=filename.substring(i+1).toLowerCase(Locale.US);
+                type = _mimeMap.get(ext);
+            }
+        }
+        return type;
+    }
+
+    /* ------------------------------------------------------------ */
+    /** Set a mime mapping
+     * @param extension
+     * @param type
+     */
+    public void addMimeMapping(String extension, String type)
+    {
+        _mimeMap.put(extension.toLowerCase(Locale.US), type);
+    }
+}

apps/i2psnark/java/src/org/klomp/snark/web/RunStandalone.java

@@ -5,7 +5,7 @@ import java.io.File;
 import net.i2p.I2PAppContext;
 import net.i2p.util.FileUtil;
 
-import org.mortbay.jetty.Server;
+import org.eclipse.jetty.server.Server;
 
 public class RunStandalone {
     static {

apps/i2psnark/java/src/org/klomp/snark/web/WriterOutputStream.java

@@ -0,0 +1,19 @@
+package org.klomp.snark.web;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.Writer;
+
+/**
+ * Treat a writer as an output stream. Quick 'n dirty, none
+ * of that "intarnasheeonaleyzayshun" stuff.  So we can treat
+ * the jsp's PrintWriter as an OutputStream
+ *
+ * @since Jetty 7 copied from routerconsole
+ */
+class WriterOutputStream extends OutputStream {
+    private final Writer _writer;
+    
+    public WriterOutputStream(Writer writer) { _writer = writer; }
+    public void write(int b) throws IOException { _writer.write(b); }
+}

apps/i2psnark/mime.properties

@@ -0,0 +1,23 @@
+7z	= application/x-7z-compressed
+ape	= audio/x-monkeys-audio
+dmg	= application/apple-diskimage
+epub	= application/epub+zip
+flac	= audio/flac
+flv	= video/x-flv
+iso	= application/x-iso9660-image
+m4a	= audio/mp4a-latm
+m4v	= video/x-m4v
+mkv	= video/x-matroska
+mp4	= video/mp4
+mpc	= audio/x-musepack
+nfo	= text/plain
+ogm	= video/ogg
+ogv	= video/ogg
+oga	= audio/ogg
+rar	= application/x-rar-compressed
+su2	= application/zip
+sud	= application/zip
+txt	= text/plain
+war	= application/java-archive
+wma	= audio/x-ms-wma
+wmv	= video/x-ms-wmv

apps/i2ptunnel/java/.classpath

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry kind="src" path="src"/>
+	<classpathentry kind="src" path="test/junit"/>
+	<classpathentry combineaccessrules="false" kind="src" path="/i2p_sdk"/>
+	<classpathentry combineaccessrules="false" kind="src" path="/ministreaming"/>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+	<classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/>
+	<classpathentry kind="output" path="build/obj"/>
+</classpath>

apps/i2ptunnel/java/.project

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>i2ptunnel</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.jdt.core.javabuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.jdt.core.javanature</nature>
+	</natures>
+</projectDescription>

apps/i2ptunnel/java/build.xml

@@ -21,6 +21,9 @@
         </depend>
     </target>
 
+    <condition property="no.bundle">
+        <isfalse value="${require.gettext}" />
+    </condition>
     <property name="javac.compilerargs" value="" />
     <property name="require.gettext" value="true" />
 
@@ -84,7 +87,7 @@
         </condition>
     </target>
 
-    <target name="bundle" depends="compile, precompilejsp">
+    <target name="bundle" depends="compile, precompilejsp" unless="no.bundle">
         <!-- Update the messages_*.po files.
              We need to supply the bat file for windows, and then change the fail property to true -->
         <exec executable="sh" osfamily="unix" failifexecutionfails="true" failonerror="${require.gettext}" >
@@ -179,6 +182,7 @@
                 <pathelement location="${ant.home}/lib/ant.jar" />
                 <pathelement location="build/i2ptunnel.jar" />
                 <pathelement location="build/temp-beans.jar" />
+                <pathelement location="../../../core/java/build/i2p.jar" />
             </classpath>
             <arg value="-d" />
             <arg value="../jsp/WEB-INF/classes" />
@@ -202,6 +206,7 @@
                 <pathelement location="../../jetty/jettylib/jsp-api.jar" />
                 <pathelement location="build/i2ptunnel.jar" />
                 <pathelement location="build/temp-beans.jar" />
+                <pathelement location="../../../core/java/build/i2p.jar" />
             </classpath>
         </javac>
         <copy file="../jsp/web.xml" tofile="../jsp/web-out.xml" />
@@ -213,6 +218,7 @@
 
     <uptodate property="precompilejsp.uptodate" targetfile="../jsp/web-out.xml">
         <srcfiles dir= "../jsp" includes="*.jsp, *.html, web.xml"/>
+        <srcfiles dir= "src/net/i2p/i2ptunnel/web" includes="*.java"/>
     </uptodate>
 
     <target name="javadoc">
@@ -230,7 +236,7 @@
         <mkdir dir="./build" />
         <mkdir dir="./build/obj" />
         <!-- We need the ant runtime, as it includes junit -->
-        <javac srcdir="./src:./test" debug="true" source="1.5" target="1.5"
+        <javac srcdir="./src:./test/junit" debug="true" source="1.5" target="1.5"
                includeAntRuntime="true"
                deprecation="on" destdir="./build/obj" >
             <compilerarg line="${javac.compilerargs}" />
@@ -248,7 +254,7 @@
                 <pathelement location="../../../core/java/build/i2p.jar" />
             </classpath>
             <batchtest>
-                <fileset dir="./test/">
+                <fileset dir="./test/junit/">
                     <include name="**/*Test.java" />
                 </fileset>
             </batchtest>

apps/i2ptunnel/java/bundle-messages.sh

@@ -1,4 +1,5 @@
-#
+#!/bin/sh
+
 # Update messages_xx.po and messages_xx.class files,
 # from both java and jsp sources.
 # Requires installed programs xgettext, msgfmt, msgmerge, and find.

apps/i2ptunnel/java/src/net/i2p/i2ptunnel/HTTPResponseOutputStream.java

@@ -54,9 +54,7 @@ class HTTPResponseOutputStream extends FilterOutputStream {
     public HTTPResponseOutputStream(OutputStream raw) {
         super(raw);
         _context = I2PAppContext.getGlobalContext();
-        _context.statManager().createRateStat("i2ptunnel.httpCompressionRatio", "ratio of compressed size to decompressed size after transfer", "I2PTunnel", new long[] { 60*60*1000 });
-        _context.statManager().createRateStat("i2ptunnel.httpCompressed", "compressed size transferred", "I2PTunnel", new long[] { 60*60*1000 });
-        _context.statManager().createRateStat("i2ptunnel.httpExpanded", "size transferred after expansion", "I2PTunnel", new long[] { 60*60*1000 });
+        // all createRateStat in I2PTunnelHTTPClient.startRunning()
         _log = _context.logManager().getLog(getClass());
         _headerBuffer = _cache.acquire();
         _buf1 = new byte[1];
@@ -67,10 +65,12 @@ class HTTPResponseOutputStream extends FilterOutputStream {
         _buf1[0] = (byte)c;
         write(_buf1, 0, 1);
     }
+
     @Override
     public void write(byte buf[]) throws IOException { 
         write(buf, 0, buf.length); 
     }
+
     @Override
     public void write(byte buf[], int off, int len) throws IOException {
         if (_headerWritten) {
@@ -183,6 +183,7 @@ class HTTPResponseOutputStream extends FilterOutputStream {
                                 _gzip = true;
                             } else if ("proxy-authenticate".equals(lcKey)) {
                                 // filter this hop-by-hop header; outproxy authentication must be configured in I2PTunnelHTTPClient
+                                // see e.g. http://blog.c22.cc/2013/03/11/privoxy-proxy-authentication-credential-exposure-cve-2013-2503/
                             } else {
                                 if ("content-length".equals(lcKey)) {
                                     // save for compress decision on server side
@@ -192,6 +193,17 @@ class HTTPResponseOutputStream extends FilterOutputStream {
                                 } else if ("content-type".equals(lcKey)) {
                                     // save for compress decision on server side
                                     _contentType = val;
+                                } else if ("set-cookie".equals(lcKey)) {
+                                    String lcVal = val.toLowerCase(Locale.US);
+                                    if (lcVal.contains("domain=b32.i2p") ||
+                                        lcVal.contains("domain=.b32.i2p")) {
+                                        // Strip privacy-damaging "supercookie" for b32.i2p
+                                        // Let's presume the user agent ignores a cookie for "i2p"
+                                        // See RFC 6265 and http://publicsuffix.org/
+                                        if (_log.shouldLog(Log.INFO))
+                                            _log.info("Stripping \"" + key + ": " + val + "\" from response ");
+                                        break;
+                                    }
                                 }
                                 out.write((key.trim() + ": " + val.trim() + "\r\n").getBytes());
                             }
@@ -274,7 +286,7 @@ class HTTPResponseOutputStream extends FilterOutputStream {
                 while ( (read = _in.read(buf)) != -1) {
                     if (_log.shouldLog(Log.DEBUG))
                         _log.debug("Read " + read + " and writing it to the browser/streams");
-;                   _out.write(buf, 0, read);
+                    _out.write(buf, 0, read);
                     _out.flush();
                     written += read;
                 }

apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnel.java

@@ -694,7 +694,7 @@ public class I2PTunnel extends EventDispatcherImpl implements Logging {
     public void runClient(String args[], Logging l) {
         boolean isShared = true;
         if (args.length >= 3)
-            isShared = Boolean.valueOf(args[2].trim()).booleanValue();
+            isShared = Boolean.parseBoolean(args[2].trim());
         if (args.length >= 2) {
             int portNum = -1;
             try {
@@ -717,7 +717,7 @@ public class I2PTunnel extends EventDispatcherImpl implements Logging {
                 addtask(task);
                 notifyEvent("clientTaskId", Integer.valueOf(task.getId()));
             } catch (IllegalArgumentException iae) {
-                String msg = "Invalid I2PTunnel configuration to create an HTTP Proxy connecting to the router at " + host + ':'+ port +
+                String msg = "Invalid I2PTunnel configuration to create a standard client tunnel connecting to the router at " + host + ':'+ port +
                              " and listening on " + listenHost + ':' + portNum;
                 _log.error(getPrefix() + msg, iae);
                 l.log(msg);
@@ -766,7 +766,7 @@ public class I2PTunnel extends EventDispatcherImpl implements Logging {
             String proxy = "";
             boolean isShared = true;
             if (args.length > 1) {
-                if (Boolean.valueOf(args[1].trim()).booleanValue()) {
+                if (Boolean.parseBoolean(args[1].trim())) {
                     isShared = true;
                     if (args.length == 3)
                         proxy = args[2];
@@ -835,7 +835,7 @@ public class I2PTunnel extends EventDispatcherImpl implements Logging {
             String proxy = "";
             boolean isShared = true;
             if (args.length > 1) {
-                if (Boolean.valueOf(args[1].trim()).booleanValue()) {
+                if (Boolean.parseBoolean(args[1].trim())) {
                     isShared = true;
                     if (args.length == 3)
                         proxy = args[2];
@@ -906,7 +906,7 @@ public class I2PTunnel extends EventDispatcherImpl implements Logging {
             
             boolean isShared = true;
             if (args.length > 2) {
-                if (Boolean.valueOf(args[2].trim()).booleanValue()) {
+                if (Boolean.parseBoolean(args[2].trim())) {
                     isShared = true;
                 } else if ("false".equalsIgnoreCase(args[2].trim())) {
                     _log.warn("args[2] == [" + args[2] + "] and rejected explicitly");
@@ -973,7 +973,7 @@ public class I2PTunnel extends EventDispatcherImpl implements Logging {
 
             boolean isShared = false;
             if (args.length > 1)
-                isShared = Boolean.valueOf(args[1].trim()).booleanValue();
+                isShared = Boolean.parseBoolean(args[1].trim());
 
             ownDest = !isShared;
             try {
@@ -1017,7 +1017,7 @@ public class I2PTunnel extends EventDispatcherImpl implements Logging {
 
             boolean isShared = false;
             if (args.length == 2)
-                isShared = Boolean.valueOf(args[1].trim()).booleanValue();
+                isShared = Boolean.parseBoolean(args[1].trim());
 
             ownDest = !isShared;
             String privateKeyFile = null;

apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelClientBase.java

@@ -190,11 +190,11 @@ public abstract class I2PTunnelClientBase extends I2PTunnelTask implements Runna
         // no need to load the netDb with leaseSets for destinations that will never 
         // be looked up
         boolean dccEnabled = (this instanceof I2PTunnelIRCClient) &&
-                      Boolean.valueOf(tunnel.getClientOptions().getProperty(I2PTunnelIRCClient.PROP_DCC)).booleanValue();
+                      Boolean.parseBoolean(tunnel.getClientOptions().getProperty(I2PTunnelIRCClient.PROP_DCC));
         if (!dccEnabled)
             tunnel.getClientOptions().setProperty("i2cp.dontPublishLeaseSet", "true");
         
-        boolean openNow = !Boolean.valueOf(tunnel.getClientOptions().getProperty("i2cp.delayOpen")).booleanValue();
+        boolean openNow = !Boolean.parseBoolean(tunnel.getClientOptions().getProperty("i2cp.delayOpen"));
         if (openNow) {
             while (sockMgr == null) {
                 verifySocketManager();
@@ -258,8 +258,8 @@ public abstract class I2PTunnelClientBase extends I2PTunnelTask implements Runna
                 if (sess == null) {
                     newManager = true;
                 } else if (sess.isClosed() &&
-                           Boolean.valueOf(getTunnel().getClientOptions().getProperty("i2cp.closeOnIdle")).booleanValue() &&
-                           Boolean.valueOf(getTunnel().getClientOptions().getProperty("i2cp.newDestOnResume")).booleanValue()) {
+                           Boolean.parseBoolean(getTunnel().getClientOptions().getProperty("i2cp.closeOnIdle")) &&
+                           Boolean.parseBoolean(getTunnel().getClientOptions().getProperty("i2cp.newDestOnResume"))) {
                     // build a new socket manager and a new dest if the session is closed.
                     getTunnel().removeSession(sess);
                     if (_log.shouldLog(Log.WARN))

apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelConnectClient.java

@@ -13,6 +13,7 @@ import java.util.Locale;
 import java.util.Properties;
 import java.util.StringTokenizer;
 
+import net.i2p.I2PAppContext;
 import net.i2p.I2PException;
 import net.i2p.client.streaming.I2PSocket;
 import net.i2p.client.streaming.I2PSocketOptions;
@@ -20,7 +21,6 @@ import net.i2p.data.Base64;
 import net.i2p.data.DataHelper;
 import net.i2p.data.Destination;
 import net.i2p.util.EventDispatcher;
-import net.i2p.util.FileUtil;
 import net.i2p.util.Log;
 import net.i2p.util.PortMapper;
 
@@ -58,6 +58,8 @@ import net.i2p.util.PortMapper;
  */
 public class I2PTunnelConnectClient extends I2PTunnelHTTPClientBase implements Runnable {
 
+    public static final String AUTH_REALM = "I2P SSL Proxy";
+
     private final static byte[] ERR_DESTINATION_UNKNOWN =
         ("HTTP/1.1 503 Service Unavailable\r\n"+
          "Content-Type: text/html; charset=iso-8859-1\r\n"+
@@ -89,17 +91,6 @@ public class I2PTunnelConnectClient extends I2PTunnelHTTPClientBase implements R
          "Your browser is misconfigured. Do not use the proxy to access the router console or other localhost destinations.<BR>")
         .getBytes();
     
-    private final static byte[] ERR_AUTH =
-        ("HTTP/1.1 407 Proxy Authentication Required\r\n"+
-         "Content-Type: text/html; charset=UTF-8\r\n"+
-         "Cache-control: no-cache\r\n"+
-         "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.5\r\n" +         // try to get a UTF-8-encoded response back for the password
-         "Proxy-Authenticate: Basic realm=\"I2P SSL Proxy\"\r\n" +
-         "\r\n"+
-         "<html><body><H1>I2P ERROR: PROXY AUTHENTICATION REQUIRED</H1>"+
-         "This proxy is configured to require authentication.<BR>")
-        .getBytes();
-
     private final static byte[] SUCCESS_RESPONSE =
         ("HTTP/1.1 200 Connection Established\r\n"+
          "Proxy-agent: I2P\r\n"+
@@ -165,6 +156,11 @@ public class I2PTunnelConnectClient extends I2PTunnelHTTPClientBase implements R
         return super.close(forced);
     }
 
+    /** @since 0.9.4 */
+    protected String getRealm() {
+        return AUTH_REALM;
+    }
+
     protected void clientConnectionRun(Socket s) {
         InputStream in = null;
         OutputStream out = null;
@@ -237,10 +233,10 @@ public class I2PTunnelConnectClient extends I2PTunnelHTTPClientBase implements R
                         _log.debug(getPrefix(requestId) + "REST  :" + restofline + ":");
                         _log.debug(getPrefix(requestId) + "DEST  :" + destination + ":");
                     }
-                } else if (line.toLowerCase(Locale.US).startsWith("proxy-authorization: basic ")) {
+                } else if (line.toLowerCase(Locale.US).startsWith("proxy-authorization: ")) {
                     // strip Proxy-Authenticate from the response in HTTPResponseOutputStream
                     // save for auth check below
-                    authorization = line.substring(27);  // "proxy-authorization: basic ".length()
+                    authorization = line.substring(21);  // "proxy-authorization: ".length()
                     line = null;
                 } else if (line.length() > 0) {
                     // Additional lines - shouldn't be too many. Firefox sends:
@@ -253,7 +249,7 @@ public class I2PTunnelConnectClient extends I2PTunnelHTTPClientBase implements R
                     line = null;
                 } else {
                     // Add Proxy-Authentication header for next hop (outproxy)
-                    if (usingWWWProxy && Boolean.valueOf(getTunnel().getClientOptions().getProperty(PROP_OUTPROXY_AUTH)).booleanValue()) {
+                    if (usingWWWProxy && Boolean.parseBoolean(getTunnel().getClientOptions().getProperty(PROP_OUTPROXY_AUTH))) {
                         // specific for this proxy
                         String user = getTunnel().getClientOptions().getProperty(PROP_OUTPROXY_USER_PREFIX + currentProxy);
                         String pw = getTunnel().getClientOptions().getProperty(PROP_OUTPROXY_PW_PREFIX + currentProxy);
@@ -281,30 +277,26 @@ public class I2PTunnelConnectClient extends I2PTunnelHTTPClientBase implements R
             }
             
             // Authorization
-            if (!authorize(s, requestId, authorization)) {
+            AuthResult result = authorize(s, requestId, method, authorization);
+            if (result != AuthResult.AUTH_GOOD) {
                 if (_log.shouldLog(Log.WARN)) {
                     if (authorization != null)
                         _log.warn(getPrefix(requestId) + "Auth failed, sending 407 again");
                     else
                         _log.warn(getPrefix(requestId) + "Auth required, sending 407");
                 }
-                writeErrorMessage(ERR_AUTH, out);
+                out.write(getAuthError(result == AuthResult.AUTH_STALE).getBytes());
                 s.close();
                 return;
             }
 
             Destination clientDest = _context.namingService().lookup(destination);
             if (clientDest == null) {
-                String str;
                 byte[] header;
                 if (usingWWWProxy)
-                    str = FileUtil.readTextFile((new File(_errorDir, "dnfp-header.ht")).getAbsolutePath(), 100, true);
+                    header = getErrorPage("dnfp-header.ht", ERR_DESTINATION_UNKNOWN);
                 else
-                    str = FileUtil.readTextFile((new File(_errorDir, "dnfh-header.ht")).getAbsolutePath(), 100, true);
-                if (str != null)
-                    header = str.getBytes();
-                else
-                    header = ERR_DESTINATION_UNKNOWN;
+                    header = getErrorPage("dnfh-header.ht", ERR_DESTINATION_UNKNOWN);
                 writeErrorMessage(header, out, targetRequest, usingWWWProxy, destination);
                 s.close();
                 return;
@@ -341,12 +333,13 @@ public class I2PTunnelConnectClient extends I2PTunnelHTTPClientBase implements R
     }
 
     private static class OnTimeout implements Runnable {
-        private Socket _socket;
-        private OutputStream _out;
-        private String _target;
-        private boolean _usingProxy;
-        private String _wwwProxy;
-        private long _requestId;
+        private final Socket _socket;
+        private final OutputStream _out;
+        private final String _target;
+        private final boolean _usingProxy;
+        private final String _wwwProxy;
+        private final long _requestId;
+
         public OnTimeout(Socket s, OutputStream out, String target, boolean usingProxy, String wwwProxy, long id) {
             _socket = s;
             _out = out;
@@ -355,6 +348,7 @@ public class I2PTunnelConnectClient extends I2PTunnelHTTPClientBase implements R
             _wwwProxy = wwwProxy;
             _requestId = id;
         }
+
         public void run() {
             //if (_log.shouldLog(Log.DEBUG))
             //    _log.debug("Timeout occured requesting " + _target);
@@ -391,17 +385,12 @@ public class I2PTunnelConnectClient extends I2PTunnelHTTPClientBase implements R
                                                   boolean usingWWWProxy, String wwwProxy, long requestId) {
         if (out == null)
             return;
+        byte[] header;
+        if (usingWWWProxy)
+            header = getErrorPage(I2PAppContext.getGlobalContext(), "dnfp-header.ht", ERR_DESTINATION_UNKNOWN);
+        else
+            header = getErrorPage(I2PAppContext.getGlobalContext(), "dnf-header.ht", ERR_DESTINATION_UNKNOWN);
         try {
-            String str;
-            byte[] header;
-            if (usingWWWProxy)
-                str = FileUtil.readTextFile((new File(_errorDir, "dnfp-header.ht")).getAbsolutePath(), 100, true);
-            else
-                str = FileUtil.readTextFile((new File(_errorDir, "dnf-header.ht")).getAbsolutePath(), 100, true);
-            if (str != null)
-                header = str.getBytes();
-            else
-                header = ERR_DESTINATION_UNKNOWN;
             writeErrorMessage(header, out, targetRequest, usingWWWProxy, wwwProxy);
         } catch (IOException ioe) {}
     }

apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPBidirProxy.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,9 +11,8 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
-
 package net.i2p.i2ptunnel;
 
 // import java.util.ArrayList;

apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPClient.java

@@ -3,9 +3,7 @@
  */
 package net.i2p.i2ptunnel;
 
-import java.io.ByteArrayOutputStream;
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -73,10 +71,14 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn
      *  via address helper links
      */
     private final ConcurrentHashMap<String, String> addressHelpers = new ConcurrentHashMap(8);
+
     /**
      *  Used to protect actions via http://proxy.i2p/
      */
     private final String _proxyNonce;
+
+    public static final String AUTH_REALM = "I2P HTTP Proxy";
+
     /**
      *  These are backups if the xxx.ht error page is missing.
      */
@@ -160,6 +162,14 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn
             "<html><body><H1>I2P ERROR: NON-HTTP PROTOCOL</H1>" +
             "The request uses a bad protocol. " +
             "The I2P HTTP Proxy supports http:// requests ONLY. Other protocols such as https:// and ftp:// are not allowed.<BR>").getBytes();
+    private final static byte[] ERR_BAD_URI =
+                                ("HTTP/1.1 403 Bad URI\r\n" +
+            "Content-Type: text/html; charset=iso-8859-1\r\n" +
+            "Cache-control: no-cache\r\n" +
+            "\r\n" +
+            "<html><body><H1>I2P ERROR: INVALID REQUEST URI</H1>" +
+            "The request URI is invalid, and probably contains illegal characters. " +
+            "If you clicked e.g. a forum link, check the end of the URI for any characters the browser has mistakenly added on.<BR>").getBytes();
     private final static byte[] ERR_LOCALHOST =
                                 ("HTTP/1.1 403 Access Denied\r\n" +
             "Content-Type: text/html; charset=iso-8859-1\r\n" +
@@ -167,15 +177,6 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn
             "\r\n" +
             "<html><body><H1>I2P ERROR: REQUEST DENIED</H1>" +
             "Your browser is misconfigured. Do not use the proxy to access the router console or other localhost destinations.<BR>").getBytes();
-    private final static byte[] ERR_AUTH =
-                                ("HTTP/1.1 407 Proxy Authentication Required\r\n" +
-            "Content-Type: text/html; charset=UTF-8\r\n" +
-            "Cache-control: no-cache\r\n" +
-            "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.5\r\n" + // try to get a UTF-8-encoded response back for the password
-            "Proxy-Authenticate: Basic realm=\"I2P HTTP Proxy\"\r\n" +
-            "\r\n" +
-            "<html><body><H1>I2P ERROR: PROXY AUTHENTICATION REQUIRED</H1>" +
-            "This proxy is configured to require authentication.<BR>").getBytes();
 
     /**
      *  This constructor always starts the tunnel (ignoring the i2cp.delayOpen option).
@@ -280,6 +281,10 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn
      */
     @Override
     public void startRunning() {
+        // following are for HTTPResponseOutputStream
+        _context.statManager().createRateStat("i2ptunnel.httpCompressionRatio", "ratio of compressed size to decompressed size after transfer", "I2PTunnel", new long[] { 60*60*1000 });
+        _context.statManager().createRateStat("i2ptunnel.httpCompressed", "compressed size transferred", "I2PTunnel", new long[] { 60*60*1000 });
+        _context.statManager().createRateStat("i2ptunnel.httpExpanded", "size transferred after expansion", "I2PTunnel", new long[] { 60*60*1000 });
         super.startRunning();
         this.isr = new InternalSocketRunner(this);
         _context.portMapper().register(PortMapper.SVC_HTTP_PROXY, getLocalPort());
@@ -300,6 +305,12 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn
         }
         return rv;
     }
+
+    /** @since 0.9.4 */
+    protected String getRealm() {
+        return AUTH_REALM;
+    }
+
     private static final String HELPER_PARAM = "i2paddresshelper";
     public static final String LOCAL_SERVER = "proxy.i2p";
     private static final boolean DEFAULT_GZIP = true;
@@ -337,6 +348,7 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn
             String userAgent = null;
             String authorization = null;
             int remotePort = 0;
+            String referer = null;
             while((line = reader.readLine(method)) != null) {
                 line = line.trim();
                 if(_log.shouldLog(Log.DEBUG)) {
@@ -424,7 +436,12 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn
                         if(_log.shouldLog(Log.WARN)) {
                             _log.warn(getPrefix(requestId) + "Bad request [" + request + "]", use);
                         }
-                        break;
+                        if(out != null) {
+                            out.write(getErrorPage("baduri", ERR_BAD_URI));
+                            writeFooter(out);
+                        }
+                        s.close();
+                        return;
                     }
                     method = params[0];
                     String protocolVersion = params[2];
@@ -521,7 +538,7 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn
                             // Try to find an address helper in the query
                             String[] helperStrings = removeHelper(query);
                             if(helperStrings != null &&
-                                    !Boolean.valueOf(getTunnel().getClientOptions().getProperty(PROP_DISABLE_HELPER)).booleanValue()) {
+                                    !Boolean.parseBoolean(getTunnel().getClientOptions().getProperty(PROP_DISABLE_HELPER))) {
                                 query = helperStrings[0];
                                 if(query.equals("")) {
                                     query = null;
@@ -736,7 +753,7 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn
                     } else if(lowercaseLine.startsWith("user-agent: ")) {
                         // save for deciding whether to offer address book form
                         userAgent = lowercaseLine.substring(12);
-                        if(!Boolean.valueOf(getTunnel().getClientOptions().getProperty(PROP_USER_AGENT)).booleanValue()) {
+                        if(!Boolean.parseBoolean(getTunnel().getClientOptions().getProperty(PROP_USER_AGENT))) {
                             line = null;
                             continue;
                         }
@@ -745,14 +762,17 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn
                         // browser to browser
                         line = null;
                         continue;
-                    } else if(lowercaseLine.startsWith("referer: ") &&
-                            !Boolean.valueOf(getTunnel().getClientOptions().getProperty(PROP_REFERER)).booleanValue()) {
-                        // Shouldn't we be more specific, like accepting in-site referers ?
-                        //line = "Referer: i2p";
-                        line = null;
-                        continue; // completely strip the line
+                    } else if (lowercaseLine.startsWith("referer: ")) {
+                        // save for address helper form below
+                        referer = line.substring(9);
+                        if (!Boolean.parseBoolean(getTunnel().getClientOptions().getProperty(PROP_REFERER))) {
+                            // Shouldn't we be more specific, like accepting in-site referers ?
+                            //line = "Referer: i2p";
+                            line = null;
+                            continue; // completely strip the line
+                        }
                     } else if(lowercaseLine.startsWith("via: ") &&
-                            !Boolean.valueOf(getTunnel().getClientOptions().getProperty(PROP_VIA)).booleanValue()) {
+                            !Boolean.parseBoolean(getTunnel().getClientOptions().getProperty(PROP_VIA))) {
                         //line = "Via: i2p";
                         line = null;
                         continue; // completely strip the line
@@ -769,10 +789,7 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn
                         // hop-by-hop header, and we definitely want to block Windows NTLM after a far-end 407.
                         // Response to far-end shouldn't happen, as we
                         // strip Proxy-Authenticate from the response in HTTPResponseOutputStream
-                        if(lowercaseLine.startsWith("proxy-authorization: basic ")) // save for auth check below
-                        {
-                            authorization = line.substring(27);  // "proxy-authorization: basic ".length()
-                        }
+                        authorization = line.substring(21);  // "proxy-authorization: ".length()
                         line = null;
                         continue;
                     } else if(lowercaseLine.startsWith("icy")) {
@@ -786,7 +803,7 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn
                     String ok = getTunnel().getClientOptions().getProperty("i2ptunnel.gzip");
                     boolean gzip = DEFAULT_GZIP;
                     if(ok != null) {
-                        gzip = Boolean.valueOf(ok).booleanValue();
+                        gzip = Boolean.parseBoolean(ok);
                     }
                     if(gzip && !usingInternalServer) {
                         // according to rfc2616 s14.3, this *should* force identity, even if
@@ -796,7 +813,7 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn
                         newRequest.append("X-Accept-Encoding: x-i2p-gzip;q=1.0, identity;q=0.5, deflate;q=0, gzip;q=0, *;q=0\r\n");
                     }
                     if(!shout) {
-                        if(!Boolean.valueOf(getTunnel().getClientOptions().getProperty(PROP_USER_AGENT)).booleanValue()) {
+                        if(!Boolean.parseBoolean(getTunnel().getClientOptions().getProperty(PROP_USER_AGENT))) {
                             // let's not advertise to external sites that we are from I2P
                             if(usingWWWProxy) {
                                 newRequest.append("User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2.6) Gecko/20100625 Firefox/3.6.6\r\n");
@@ -806,7 +823,7 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn
                         }
                     }
                     // Add Proxy-Authentication header for next hop (outproxy)
-                    if(usingWWWProxy && Boolean.valueOf(getTunnel().getClientOptions().getProperty(PROP_OUTPROXY_AUTH)).booleanValue()) {
+                    if(usingWWWProxy && Boolean.parseBoolean(getTunnel().getClientOptions().getProperty(PROP_OUTPROXY_AUTH))) {
                         // specific for this proxy
                         String user = getTunnel().getClientOptions().getProperty(PROP_OUTPROXY_USER_PREFIX + currentProxy);
                         String pw = getTunnel().getClientOptions().getProperty(PROP_OUTPROXY_PW_PREFIX + currentProxy);
@@ -850,7 +867,8 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn
             }
 
             // Authorization
-            if(!authorize(s, requestId, authorization)) {
+            AuthResult result = authorize(s, requestId, method, authorization);
+            if (result != AuthResult.AUTH_GOOD) {
                 if(_log.shouldLog(Log.WARN)) {
                     if(authorization != null) {
                         _log.warn(getPrefix(requestId) + "Auth failed, sending 407 again");
@@ -858,7 +876,7 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn
                         _log.warn(getPrefix(requestId) + "Auth required, sending 407");
                     }
                 }
-                out.write(getErrorPage("auth", ERR_AUTH));
+                out.write(getAuthError(result == AuthResult.AUTH_STALE).getBytes());
                 writeFooter(out);
                 s.close();
                 return;
@@ -869,7 +887,7 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn
             if(usingInternalServer) {
                 // disable the add form if address helper is disabled
                 if(internalPath.equals("/add") &&
-                        Boolean.valueOf(getTunnel().getClientOptions().getProperty(PROP_DISABLE_HELPER)).booleanValue()) {
+                        Boolean.parseBoolean(getTunnel().getClientOptions().getProperty(PROP_DISABLE_HELPER))) {
                     out.write(ERR_HELPER_DISABLED);
                 } else {
                     LocalHTTPServer.serveLocalFile(out, method, internalPath, internalRawQuery, _proxyNonce);
@@ -949,8 +967,8 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn
             // Don't do this for eepget, which uses a user-agent of "Wget"
             if(ahelperNew && "GET".equals(method) &&
                     (userAgent == null || !userAgent.startsWith("Wget")) &&
-                    !Boolean.valueOf(getTunnel().getClientOptions().getProperty(PROP_DISABLE_HELPER)).booleanValue()) {
-                writeHelperSaveForm(out, destination, ahelperKey, targetRequest);
+                    !Boolean.parseBoolean(getTunnel().getClientOptions().getProperty(PROP_DISABLE_HELPER))) {
+                writeHelperSaveForm(out, destination, ahelperKey, targetRequest, referer);
                 s.close();
                 return;
             }
@@ -1014,7 +1032,8 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn
     }
 
     /** @since 0.8.7 */
-    private void writeHelperSaveForm(OutputStream out, String destination, String ahelperKey, String targetRequest) throws IOException {
+    private void writeHelperSaveForm(OutputStream out, String destination, String ahelperKey,
+                                     String targetRequest, String referer) throws IOException {
         if(out == null) {
             return;
         }
@@ -1045,6 +1064,10 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn
             out.write(("<br><button type=\"submit\" name=\"master\" value=\"master\">" + _("Save {0} to master address book and continue to eepsite", destination) + "</button><br>\n").getBytes("UTF-8"));
             out.write(("<button type=\"submit\" name=\"private\" value=\"private\">" + _("Save {0} to private address book and continue to eepsite", destination) + "</button>\n").getBytes("UTF-8"));
         }
+        // Firefox (and others?) don't send referer to meta refresh target, which is
+        // what the jump servers use, so this isn't that useful.
+        if (referer != null)
+            out.write(("<input type=\"hidden\" name=\"referer\" value=\"" + referer + "\">\n").getBytes("UTF-8"));
         out.write(("<input type=\"hidden\" name=\"url\" value=\"" + targetRequest + "\">\n" +
                 "</form></div></div>").getBytes());
         writeFooter(out);
@@ -1095,61 +1118,6 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn
         return Base32.encode(_dest.calculateHash().getData()) + ".b32.i2p";
     }
 
-    /**
-     *  foo => errordir/foo-header_xx.ht for lang xx, or errordir/foo-header.ht,
-     *  or the backup byte array on fail.
-     *
-     *  .ht files must be UTF-8 encoded and use \r\n terminators so the
-     *  HTTP headers are conformant.
-     *  We can't use FileUtil.readFile() because it strips \r
-     *
-     *  @return non-null
-     */
-    private byte[] getErrorPage(String base, byte[] backup) {
-        return getErrorPage(_context, base, backup);
-    }
-
-    private static byte[] getErrorPage(I2PAppContext ctx, String base, byte[] backup) {
-        File errorDir = new File(ctx.getBaseDir(), "docs");
-        String lang = ctx.getProperty("routerconsole.lang", Locale.getDefault().getLanguage());
-        if(lang != null && lang.length() > 0 && !lang.equals("en")) {
-            File file = new File(errorDir, base + "-header_" + lang + ".ht");
-            try {
-                return readFile(file);
-            } catch(IOException ioe) {
-                // try the english version now
-            }
-        }
-        File file = new File(errorDir, base + "-header.ht");
-        try {
-            return readFile(file);
-        } catch(IOException ioe) {
-            return backup;
-        }
-    }
-
-    private static byte[] readFile(File file) throws IOException {
-        FileInputStream fis = null;
-        byte[] buf = new byte[512];
-        ByteArrayOutputStream baos = new ByteArrayOutputStream(2048);
-        try {
-            int len = 0;
-            fis = new FileInputStream(file);
-            while((len = fis.read(buf)) > 0) {
-                baos.write(buf, 0, len);
-            }
-            return baos.toByteArray();
-        } finally {
-            try {
-                if(fis != null) {
-                    fis.close();
-                }
-            } catch(IOException foo) {
-            }
-        }
-    // we won't ever get here
-    }
-
     /**
      *  Public only for LocalHTTPServer, not for general use
      */
@@ -1163,12 +1131,12 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn
 
     private static class OnTimeout implements Runnable {
 
-        private Socket _socket;
-        private OutputStream _out;
-        private String _target;
-        private boolean _usingProxy;
-        private String _wwwProxy;
-        private long _requestId;
+        private final Socket _socket;
+        private final OutputStream _out;
+        private final String _target;
+        private final boolean _usingProxy;
+        private final String _wwwProxy;
+        private final long _requestId;
 
         public OnTimeout(Socket s, OutputStream out, String target, boolean usingProxy, String wwwProxy, long id) {
             _socket = s;

apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPClientBase.java

@@ -3,19 +3,30 @@
  */
 package net.i2p.i2ptunnel;
 
+import java.io.ByteArrayOutputStream;
+import java.io.FileInputStream;
+import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.net.Socket;
 import java.util.ArrayList;
 import java.io.File;
+import java.util.BitSet;
+import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import net.i2p.I2PAppContext;
 import net.i2p.client.streaming.I2PSocketManager;
 import net.i2p.data.Base64;
+import net.i2p.data.DataHelper;
 import net.i2p.util.EventDispatcher;
 import net.i2p.util.InternalSocket;
 import net.i2p.util.Log;
+import net.i2p.util.PasswordManager;
 
 /**
  * Common things for HTTPClient and ConnectClient
@@ -25,6 +36,26 @@ import net.i2p.util.Log;
  */
 public abstract class I2PTunnelHTTPClientBase extends I2PTunnelClientBase implements Runnable {
 
+    private static final int PROXYNONCE_BYTES = 8;
+    private static final int MD5_BYTES = 16;
+    /** 24 */
+    private static final int NONCE_BYTES = DataHelper.DATE_LENGTH + MD5_BYTES;
+    private static final long MAX_NONCE_AGE = 60*60*1000L;
+    private static final int MAX_NONCE_COUNT = 1024;
+
+    private static final String ERR_AUTH1 =
+            "HTTP/1.1 407 Proxy Authentication Required\r\n" +
+            "Content-Type: text/html; charset=UTF-8\r\n" +
+            "Cache-control: no-cache\r\n" +
+            "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.5\r\n" + // try to get a UTF-8-encoded response back for the password
+            "Proxy-Authenticate: ";
+    // put the auth type and realm in between
+    private static final String ERR_AUTH2 =
+            "\r\n" +
+            "\r\n" +
+            "<html><body><H1>I2P ERROR: PROXY AUTHENTICATION REQUIRED</H1>" +
+            "This proxy is configured to require authentication.";
+
     protected final List<String> _proxyList;
 
     protected final static byte[] ERR_NO_OUTPROXY =
@@ -40,7 +71,9 @@ public abstract class I2PTunnelHTTPClientBase extends I2PTunnelClientBase implem
     /** used to assign unique IDs to the threads / clients.  no logic or functionality */
     protected static volatile long __clientId = 0;
 
-    protected static final File _errorDir = new File(I2PAppContext.getGlobalContext().getBaseDir(), "docs");
+    private final byte[] _proxyNonce;
+    private final ConcurrentHashMap<String, NonceInfo> _nonces;
+    private final AtomicInteger _nonceCleanCounter = new AtomicInteger();
 
     protected String getPrefix(long requestId) { return "Client[" + _clientId + "/" + requestId + "]: "; }
     
@@ -63,6 +96,9 @@ public abstract class I2PTunnelHTTPClientBase extends I2PTunnelClientBase implem
                                I2PTunnel tunnel) throws IllegalArgumentException {
         super(localPort, ownDest, l, notifyThis, handlerName, tunnel);
         _proxyList = new ArrayList(4);
+        _proxyNonce = new byte[PROXYNONCE_BYTES];
+        _context.random().nextBytes(_proxyNonce);
+        _nonces = new ConcurrentHashMap();
     }
 
     /**
@@ -76,8 +112,13 @@ public abstract class I2PTunnelHTTPClientBase extends I2PTunnelClientBase implem
             throws IllegalArgumentException {
         super(localPort, l, sktMgr, tunnel, notifyThis, clientId);
         _proxyList = new ArrayList(4);
+        _proxyNonce = new byte[PROXYNONCE_BYTES];
+        _context.random().nextBytes(_proxyNonce);
+        _nonces = new ConcurrentHashMap();
     }
 
+    //////// Authorization stuff
+
     /** all auth @since 0.8.2 */
     public static final String PROP_AUTH = "proxyAuth";
     public static final String PROP_USER = "proxyUsername";
@@ -90,68 +131,410 @@ public abstract class I2PTunnelHTTPClientBase extends I2PTunnelClientBase implem
     /** passwords for specific outproxies may be added with outproxyUsername.fooproxy.i2p=user and outproxyPassword.fooproxy.i2p=pw */
     public static final String PROP_OUTPROXY_USER_PREFIX = PROP_OUTPROXY_USER + '.';
     public static final String PROP_OUTPROXY_PW_PREFIX = PROP_OUTPROXY_PW + '.';
+    /** new style MD5 auth */
+    public static final String PROP_PROXY_DIGEST_PREFIX = "proxy.auth.";
+    public static final String PROP_PROXY_DIGEST_SUFFIX = ".md5";
+    public static final String BASIC_AUTH = "basic";
+    public static final String DIGEST_AUTH = "digest";
+
+    protected abstract String getRealm();
+
+    protected enum AuthResult {AUTH_BAD_REQ, AUTH_BAD, AUTH_STALE, AUTH_GOOD}
 
     /**
-     *  @param authorization may be null
-     *  @return success
+     *  @since 0.9.6
      */
-    protected boolean authorize(Socket s, long requestId, String authorization) {
-        // Authorization
-        // Ref: RFC 2617
-        // If the socket is an InternalSocket, no auth required.
-        String authRequired = getTunnel().getClientOptions().getProperty(PROP_AUTH);
-        if (Boolean.valueOf(authRequired).booleanValue() ||
-            (authRequired != null && "basic".equals(authRequired.toLowerCase(Locale.US)))) {
-            if (s instanceof InternalSocket) {
-                if (_log.shouldLog(Log.INFO))
-                    _log.info(getPrefix(requestId) + "Internal access, no auth required");
-                return true;
-            } else if (authorization != null) {
-                // hmm safeDecode(foo, true) to use standard alphabet is private in Base64
-                byte[] decoded = Base64.decode(authorization.replace("/", "~").replace("+", "="));
-                if (decoded != null) {
-                    // We send Accept-Charset: UTF-8 in the 407 so hopefully it comes back that way inside the B64 ?
-                    try {
-                        String dec = new String(decoded, "UTF-8");
-                        String[] parts = dec.split(":");
-                        String user = parts[0];
-                        String pw = parts[1];
-                        // first try pw for that user
-                        String configPW = getTunnel().getClientOptions().getProperty(PROP_PW_PREFIX + user);
-                        if (configPW == null) {
-                            // if not, look at default user and pw
-                            String configUser = getTunnel().getClientOptions().getProperty(PROP_USER);
-                            if (user.equals(configUser))
-                                configPW = getTunnel().getClientOptions().getProperty(PROP_PW);
-                        }
-                        if (configPW != null) {
-                            if (pw.equals(configPW)) {
-                                if (_log.shouldLog(Log.INFO))
-                                    _log.info(getPrefix(requestId) + "Good auth - user: " + user + " pw: " + pw);
-                                return true;
-                            } else {
-                                if (_log.shouldLog(Log.WARN))
-                                    _log.warn(getPrefix(requestId) + "Bad auth, pw mismatch - user: " + user + " pw: " + pw + " expected: " + configPW);
-                            }
-                        } else {
-                            if (_log.shouldLog(Log.WARN))
-                                _log.warn(getPrefix(requestId) + "Bad auth, no stored pw for user: " + user + " pw: " + pw);
-                        }
-                    } catch (UnsupportedEncodingException uee) {
-                        _log.error(getPrefix(requestId) + "No UTF-8 support? B64: " + authorization, uee);
-                    } catch (ArrayIndexOutOfBoundsException aioobe) {
-                        // no ':' in response
-                        if (_log.shouldLog(Log.WARN))
-                            _log.warn(getPrefix(requestId) + "Bad auth B64: " + authorization, aioobe);
-                    }
-                } else {
-                    if (_log.shouldLog(Log.WARN))
-                        _log.warn(getPrefix(requestId) + "Bad auth B64: " + authorization);
-                }
+    private static class NonceInfo {
+        private final long expires;
+        private final BitSet counts;
+
+        public NonceInfo(long exp) {
+            expires = exp;
+            counts = new BitSet(MAX_NONCE_COUNT);
+        }
+
+        public long getExpires() {
+            return expires;
+        }
+
+        public AuthResult isValid(int nc) {
+            if (nc <= 0)
+                return AuthResult.AUTH_BAD;
+            if (nc >= MAX_NONCE_COUNT)
+                return AuthResult.AUTH_STALE;
+            synchronized(counts) {
+                if (counts.get(nc))
+                    return AuthResult.AUTH_BAD;
+                counts.set(nc);
             }
-            return false;
-        } else {
-            return true;
+            return AuthResult.AUTH_GOOD;
         }
     }
+
+    /**
+     *  @since 0.9.4
+     */
+    protected boolean isDigestAuthRequired() {
+        String authRequired = getTunnel().getClientOptions().getProperty(PROP_AUTH);
+        if (authRequired == null)
+            return false;
+        return authRequired.toLowerCase(Locale.US).equals("digest");
+    }
+
+    /**
+     *  Authorization
+     *  Ref: RFC 2617
+     *  If the socket is an InternalSocket, no auth required.
+     *
+     *  @param method GET, POST, etc.
+     *  @param authorization may be null, the full auth line e.g. "Basic lskjlksjf"
+     *  @return success
+     */
+    protected AuthResult authorize(Socket s, long requestId, String method, String authorization) {
+        String authRequired = getTunnel().getClientOptions().getProperty(PROP_AUTH);
+        if (authRequired == null)
+            return AuthResult.AUTH_GOOD;
+        authRequired = authRequired.toLowerCase(Locale.US);
+        if (authRequired.equals("false"))
+            return AuthResult.AUTH_GOOD;
+        if (s instanceof InternalSocket) {
+            if (_log.shouldLog(Log.INFO))
+                _log.info(getPrefix(requestId) + "Internal access, no auth required");
+            return AuthResult.AUTH_GOOD;
+        }
+        if (authorization == null)
+            return AuthResult.AUTH_BAD;
+        if (_log.shouldLog(Log.INFO))
+            _log.info(getPrefix(requestId) + "Auth: " + authorization);
+        String authLC = authorization.toLowerCase(Locale.US);
+        if (authRequired.equals("true") || authRequired.equals(BASIC_AUTH)) {
+            if (!authLC.startsWith("basic "))
+                return AuthResult.AUTH_BAD;
+            authorization = authorization.substring(6);
+
+            // hmm safeDecode(foo, true) to use standard alphabet is private in Base64
+            byte[] decoded = Base64.decode(authorization.replace("/", "~").replace("+", "="));
+            if (decoded != null) {
+                // We send Accept-Charset: UTF-8 in the 407 so hopefully it comes back that way inside the B64 ?
+                try {
+                    String dec = new String(decoded, "UTF-8");
+                    String[] parts = dec.split(":");
+                    String user = parts[0];
+                    String pw = parts[1];
+                    // first try pw for that user
+                    String configPW = getTunnel().getClientOptions().getProperty(PROP_PW_PREFIX + user);
+                    if (configPW == null) {
+                        // if not, look at default user and pw
+                        String configUser = getTunnel().getClientOptions().getProperty(PROP_USER);
+                        if (user.equals(configUser))
+                            configPW = getTunnel().getClientOptions().getProperty(PROP_PW);
+                    }
+                    if (configPW != null) {
+                        if (pw.equals(configPW)) {
+                            if (_log.shouldLog(Log.INFO))
+                                _log.info(getPrefix(requestId) + "Good auth - user: " + user + " pw: " + pw);
+                            return AuthResult.AUTH_GOOD;
+                        }
+                    }
+                    _log.logAlways(Log.WARN, "PROXY AUTH FAILURE: user " + user);
+                } catch (UnsupportedEncodingException uee) {
+                    _log.error(getPrefix(requestId) + "No UTF-8 support? B64: " + authorization, uee);
+                } catch (ArrayIndexOutOfBoundsException aioobe) {
+                    // no ':' in response
+                    if (_log.shouldLog(Log.WARN))
+                        _log.warn(getPrefix(requestId) + "Bad auth B64: " + authorization, aioobe);
+                    return AuthResult.AUTH_BAD_REQ;
+                }
+                return AuthResult.AUTH_BAD;
+            } else {
+                if (_log.shouldLog(Log.WARN))
+                    _log.warn(getPrefix(requestId) + "Bad auth B64: " + authorization);
+                return AuthResult.AUTH_BAD_REQ;
+            }
+        } else if (authRequired.equals(DIGEST_AUTH)) {
+            if (!authLC.startsWith("digest "))
+                return AuthResult.AUTH_BAD;
+            authorization = authorization.substring(7);
+            Map<String, String> args = parseArgs(authorization);
+            AuthResult rv = validateDigest(method, args);
+            return rv;
+        } else {
+            _log.error("Unknown proxy authorization type configured: " + authRequired);
+            return AuthResult.AUTH_BAD_REQ;
+        }
+    }
+
+    /**
+     *  Verify all of it.
+     *  Ref: RFC 2617
+     *  @since 0.9.4
+     */
+    private AuthResult validateDigest(String method, Map<String, String> args) {
+        String user = args.get("username");
+        String realm = args.get("realm");
+        String nonce = args.get("nonce");
+        String qop = args.get("qop");
+        String uri = args.get("uri");
+        String cnonce = args.get("cnonce");
+        String nc = args.get("nc");
+        String response = args.get("response");
+        if (user == null || realm == null || nonce == null || qop == null ||
+            uri == null || cnonce == null || nc == null || response == null) {
+            if (_log.shouldLog(Log.INFO))
+                _log.info("Bad digest request: " + DataHelper.toString(args));
+            return AuthResult.AUTH_BAD_REQ;
+        }
+        // nonce check
+        AuthResult check = verifyNonce(nonce, nc);
+        if (check != AuthResult.AUTH_GOOD) {
+            if (_log.shouldLog(Log.INFO))
+                _log.info("Bad digest nonce: " + check + ' ' + DataHelper.toString(args));
+            return check;
+        }
+        // get H(A1) == stored password
+        String ha1 = getTunnel().getClientOptions().getProperty(PROP_PROXY_DIGEST_PREFIX + user +
+                                                                PROP_PROXY_DIGEST_SUFFIX);
+        if (ha1 == null) {
+            _log.logAlways(Log.WARN, "PROXY AUTH FAILURE: user " + user);
+            return AuthResult.AUTH_BAD;
+        }
+        // get H(A2)
+        String a2 = method + ':' + uri;
+        String ha2 = PasswordManager.md5Hex(a2);
+        // response check
+        String kd = ha1 + ':' + nonce + ':' + nc + ':' + cnonce + ':' + qop + ':' + ha2;
+        String hkd = PasswordManager.md5Hex(kd);
+        if (!response.equals(hkd)) {
+            _log.logAlways(Log.WARN, "PROXY AUTH FAILURE: user " + user);
+            if (_log.shouldLog(Log.INFO))
+                _log.info("Bad digest auth: " + DataHelper.toString(args));
+            return AuthResult.AUTH_BAD;
+        }
+        if (_log.shouldLog(Log.INFO))
+            _log.info("Good digest auth - user: " + user);
+        return AuthResult.AUTH_GOOD;
+    }
+
+    /**
+     *  The Base 64 of 24 bytes: (now, md5 of (now, proxy nonce))
+     *  @since 0.9.4
+     */
+    private String getNonce() {
+        byte[] b = new byte[DataHelper.DATE_LENGTH + PROXYNONCE_BYTES];
+        byte[] n = new byte[NONCE_BYTES];
+        long now = _context.clock().now();
+        DataHelper.toLong(b, 0, DataHelper.DATE_LENGTH, now);
+        System.arraycopy(_proxyNonce, 0, b, DataHelper.DATE_LENGTH, PROXYNONCE_BYTES);
+        System.arraycopy(b, 0, n, 0, DataHelper.DATE_LENGTH);
+        byte[] md5 = PasswordManager.md5Sum(b);
+        System.arraycopy(md5, 0, n, DataHelper.DATE_LENGTH, MD5_BYTES);
+        String rv = Base64.encode(n);
+        _nonces.putIfAbsent(rv, new NonceInfo(now + MAX_NONCE_AGE));
+        return rv;
+    }
+
+    /**
+     *  Verify the Base 64 of 24 bytes: (now, md5 of (now, proxy nonce))
+     *  and the nonce count.
+     *  @param b64 nonce non-null
+     *  @param ncs nonce count string non-null
+     *  @since 0.9.4
+     */
+    private AuthResult verifyNonce(String b64, String ncs) {
+        if (_nonceCleanCounter.incrementAndGet() % 16 == 0)
+            cleanNonces();
+        byte[] n = Base64.decode(b64);
+        if (n == null || n.length != NONCE_BYTES)
+            return AuthResult.AUTH_BAD;
+        long now = _context.clock().now();
+        long stamp = DataHelper.fromLong(n, 0, DataHelper.DATE_LENGTH);
+        if (now - stamp > MAX_NONCE_AGE) {
+            _nonces.remove(b64);
+            return AuthResult.AUTH_STALE;
+        }
+        NonceInfo info = _nonces.get(b64);
+        if (info == null)
+            return AuthResult.AUTH_STALE;
+        byte[] b = new byte[DataHelper.DATE_LENGTH + PROXYNONCE_BYTES];
+        System.arraycopy(n, 0, b, 0, DataHelper.DATE_LENGTH);
+        System.arraycopy(_proxyNonce, 0, b, DataHelper.DATE_LENGTH, PROXYNONCE_BYTES);
+        byte[] md5 = PasswordManager.md5Sum(b);
+        if (!DataHelper.eq(md5, 0, n, DataHelper.DATE_LENGTH, MD5_BYTES))
+            return AuthResult.AUTH_BAD;
+        try {
+            int nc = Integer.parseInt(ncs, 16);
+            return info.isValid(nc);
+        } catch (NumberFormatException nfe) {
+            return AuthResult.AUTH_BAD;
+        }
+    }
+
+
+    /**
+     *  Remove expired nonces from map
+     *  @since 0.9.6
+     */
+    private void cleanNonces() {
+        long now = _context.clock().now();
+        for (Iterator<NonceInfo> iter = _nonces.values().iterator(); iter.hasNext(); ) {
+            NonceInfo info = iter.next();
+            if (info.getExpires() <= now)
+                iter.remove();
+        }
+    }
+
+    /**
+     *  What to send if digest auth fails
+     *  @since 0.9.4
+     */
+    protected String getAuthError(boolean isStale) {
+        boolean isDigest = isDigestAuthRequired();
+        return
+            ERR_AUTH1 +
+            (isDigest ? "Digest" : "Basic") +
+            " realm=\"" + getRealm() + '"' +
+            (isDigest ? ", nonce=\"" + getNonce() + "\"," +
+                        " algorithm=MD5," +
+                        " qop=\"auth\"" +
+                        (isStale ? ", stale=true" : "")
+                      : "") +
+            ERR_AUTH2;
+    }
+
+    /**
+     *  Modified from LoadClientAppsJob.
+     *  All keys are mapped to lower case.
+     *  Ref: RFC 2617
+     *
+     *  @param args non-null
+     *  @since 0.9.4
+     */
+    private static Map<String, String> parseArgs(String args) {
+        Map<String, String> rv = new HashMap(8);
+        char data[] = args.toCharArray();
+        StringBuilder buf = new StringBuilder(32);
+        boolean isQuoted = false;
+        String key = null;
+        for (int i = 0; i < data.length; i++) {
+            switch (data[i]) {
+                case '\"':
+                    if (isQuoted) {
+                        // keys never quoted
+                        if (key != null) {
+                            rv.put(key, buf.toString().trim());
+                            key = null;
+                        }
+                        buf.setLength(0);
+                    }
+                    isQuoted = !isQuoted;
+                    break;
+
+                case ' ':
+                case '\r':
+                case '\n':
+                case '\t':
+                case ',':
+                    // whitespace - if we're in a quoted section, keep this as part of the quote,
+                    // otherwise use it as a delim
+                    if (isQuoted) {
+                        buf.append(data[i]);
+                    } else {
+                        if (key != null) {
+                            rv.put(key, buf.toString().trim());
+                            key = null;
+                        }
+                        buf.setLength(0);
+                    }
+                    break;
+
+                case '=':
+                    if (isQuoted) {
+                        buf.append(data[i]);
+                    } else {
+                        key = buf.toString().trim().toLowerCase(Locale.US);
+                        buf.setLength(0);
+                    }
+                    break;
+
+                default:
+                    buf.append(data[i]);
+                    break;
+            }
+        }
+        if (key != null)
+            rv.put(key, buf.toString().trim());
+        return rv;
+    }
+
+    //////// Error page stuff
+
+    /**
+     *  foo => errordir/foo-header_xx.ht for lang xx, or errordir/foo-header.ht,
+     *  or the backup byte array on fail.
+     *
+     *  .ht files must be UTF-8 encoded and use \r\n terminators so the
+     *  HTTP headers are conformant.
+     *  We can't use FileUtil.readFile() because it strips \r
+     *
+     *  @return non-null
+     *  @since 0.9.4 moved from I2PTunnelHTTPClient
+     */
+    protected byte[] getErrorPage(String base, byte[] backup) {
+        return getErrorPage(_context, base, backup);
+    }
+
+    /**
+     *  foo => errordir/foo-header_xx.ht for lang xx, or errordir/foo-header.ht,
+     *  or the backup byte array on fail.
+     *
+     *  .ht files must be UTF-8 encoded and use \r\n terminators so the
+     *  HTTP headers are conformant.
+     *  We can't use FileUtil.readFile() because it strips \r
+     *
+     *  @return non-null
+     *  @since 0.9.4 moved from I2PTunnelHTTPClient
+     */
+    protected static byte[] getErrorPage(I2PAppContext ctx, String base, byte[] backup) {
+        File errorDir = new File(ctx.getBaseDir(), "docs");
+        String lang = ctx.getProperty("routerconsole.lang", Locale.getDefault().getLanguage());
+        if(lang != null && lang.length() > 0 && !lang.equals("en")) {
+            File file = new File(errorDir, base + "-header_" + lang + ".ht");
+            try {
+                return readFile(file);
+            } catch(IOException ioe) {
+                // try the english version now
+            }
+        }
+        File file = new File(errorDir, base + "-header.ht");
+        try {
+            return readFile(file);
+        } catch(IOException ioe) {
+            return backup;
+        }
+    }
+
+    /**
+     *  @since 0.9.4 moved from I2PTunnelHTTPClient
+     */
+    private static byte[] readFile(File file) throws IOException {
+        FileInputStream fis = null;
+        byte[] buf = new byte[2048];
+        ByteArrayOutputStream baos = new ByteArrayOutputStream(2048);
+        try {
+            int len = 0;
+            fis = new FileInputStream(file);
+            while((len = fis.read(buf)) > 0) {
+                baos.write(buf, 0, len);
+            }
+            return baos.toByteArray();
+        } finally {
+            try {
+                if(fis != null) {
+                    fis.close();
+                }
+            } catch(IOException foo) {
+            }
+        }
+        // we won't ever get here
+    }
 }

apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPServer.java

@@ -45,7 +45,8 @@ public class I2PTunnelHTTPServer extends I2PTunnelServer {
     private static final String[] CLIENT_SKIPHEADERS = {HASH_HEADER, DEST64_HEADER, DEST32_HEADER};
     private static final String SERVER_HEADER = "Server";
     private static final String[] SERVER_SKIPHEADERS = {SERVER_HEADER};
-    private static final long HEADER_TIMEOUT = 60*1000;
+    private static final long HEADER_TIMEOUT = 15*1000;
+    private static final long TOTAL_HEADER_TIMEOUT = 2 * HEADER_TIMEOUT;
     private static final long START_INTERVAL = (60 * 1000) * 3;
     private long _startedOn = 0L;
 
@@ -154,7 +155,7 @@ public class I2PTunnelHTTPServer extends I2PTunnelServer {
             boolean allowGZIP = true;
             if (opts != null) {
                 String val = opts.getProperty("i2ptunnel.gzip");
-                if ( (val != null) && (!Boolean.valueOf(val).booleanValue()) ) 
+                if ( (val != null) && (!Boolean.parseBoolean(val)) ) 
                     allowGZIP = false;
             }
             if (_log.shouldLog(Log.INFO))
@@ -492,7 +493,8 @@ public class I2PTunnelHTTPServer extends I2PTunnelServer {
     	}
     }
 
-    protected static Map<String, List<String>> readHeaders(InputStream in, StringBuilder command, String[] skipHeaders, I2PAppContext ctx) throws IOException {
+    protected static Map<String, List<String>> readHeaders(InputStream in, StringBuilder command,
+                                                           String[] skipHeaders, I2PAppContext ctx) throws IOException {
     	HashMap<String, List<String>> headers = new HashMap<String, List<String>>();
         StringBuilder buf = new StringBuilder(128);
         
@@ -516,6 +518,8 @@ public class I2PTunnelHTTPServer extends I2PTunnelServer {
         if (trimmed > 0)
             ctx.statManager().addRateData("i2ptunnel.httpNullWorkaround", trimmed, 0);
         
+        // slowloris / darkloris
+        long expire = ctx.clock().now() + TOTAL_HEADER_TIMEOUT;
         int i = 0;
         while (true) {
             if (++i > MAX_HEADERS)
@@ -528,6 +532,8 @@ public class I2PTunnelHTTPServer extends I2PTunnelServer {
                 // end of headers reached
                 return headers;
             } else {
+                if (ctx.clock().now() > expire)
+                    throw new IOException("Headers took too long [" + buf.toString() + "]");
                 int split = buf.indexOf(":");
                 if (split <= 0) throw new IOException("Invalid HTTP header, missing colon [" + buf.toString() + "]");
                 String name = buf.substring(0, split).trim();

apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelIRCClient.java

@@ -87,7 +87,7 @@ public class I2PTunnelIRCClient extends I2PTunnelClientBase {
         
         setName("IRC Client on " + tunnel.listenHost + ':' + localPort);
 
-        _dccEnabled = Boolean.valueOf(tunnel.getClientOptions().getProperty(PROP_DCC)).booleanValue();
+        _dccEnabled = Boolean.parseBoolean(tunnel.getClientOptions().getProperty(PROP_DCC));
         // TODO add some prudent tunnel options (or is it too late?)
 
         startRunning();

apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelIRCServer.java

@@ -62,7 +62,8 @@ public class I2PTunnelIRCServer extends I2PTunnelServer implements Runnable {
 	public static final String PROP_WEBIRC_SPOOF_IP_DEFAULT="127.0.0.1";
     public static final String PROP_HOSTNAME="ircserver.fakeHostname";
     public static final String PROP_HOSTNAME_DEFAULT="%f.b32.i2p";
-    private static final long HEADER_TIMEOUT = 60*1000;
+    private static final long HEADER_TIMEOUT = 15*1000;
+    private static final long TOTAL_HEADER_TIMEOUT = 2 * HEADER_TIMEOUT;
     
     private final static byte[] ERR_UNAVAILABLE =
         (":ircserver.i2p 499 you :" +
@@ -71,6 +72,11 @@ public class I2PTunnelIRCServer extends I2PTunnelServer implements Runnable {
          "\r\n")
          .getBytes();
 
+    private static final String[] BAD_PROTOCOLS = {
+        "GET ", "HEAD ", "POST ", "GNUTELLA CONNECT", "\023BitTorrent protocol"
+    };
+
+
     /**
      * @throws IllegalArgumentException if the I2PTunnel does not contain
      *                                  valid config to contact the router
@@ -78,11 +84,9 @@ public class I2PTunnelIRCServer extends I2PTunnelServer implements Runnable {
 
     public I2PTunnelIRCServer(InetAddress host, int port, File privkey, String privkeyname, Logging l, EventDispatcher notifyThis, I2PTunnel tunnel) {
         super(host, port, privkey, privkeyname, l, notifyThis, tunnel);
-        initCloak(tunnel);
-    }
 
-    /** generate a random 32 bytes, or the hash of the passphrase */
-    private void initCloak(I2PTunnel tunnel) {
+        // generate a random 32 bytes, or the hash of the passphrase
+
         // get the properties of this server-tunnel
         Properties opts = tunnel.getClientOptions();
         
@@ -188,12 +192,22 @@ public class I2PTunnelIRCServer extends I2PTunnelServer implements Runnable {
         StringBuilder buf = new StringBuilder(128);
         int lineCount = 0;
         
+        // slowloris / darkloris
+        long expire = System.currentTimeMillis() + TOTAL_HEADER_TIMEOUT;
         while (true) {
             String s = DataHelper.readLine(in);
             if (s == null)
                 throw new IOException("EOF reached before the end of the headers [" + buf.toString() + "]");
+            if (lineCount == 0) {
+                for (int i = 0; i < BAD_PROTOCOLS.length; i++) {
+                    if (s.startsWith(BAD_PROTOCOLS[i]))
+                        throw new IOException("Bad protocol " + BAD_PROTOCOLS[i]);
+                }
+            }
             if (++lineCount > 10)
                 throw new IOException("Too many lines before USER or SERVER, giving up");
+            if (System.currentTimeMillis() > expire)
+                throw new IOException("Headers took too long [" + buf.toString() + "]");
             s = s.trim();
             //if (_log.shouldLog(Log.DEBUG))
             //    _log.debug("Got line: " + s);
@@ -231,9 +245,9 @@ public class I2PTunnelIRCServer extends I2PTunnelServer implements Runnable {
         return buf.toString();
     }
     
-    private byte[] cloakKey; // 32 bytes of stuff to scramble the dest with
-    private String hostname;
-    private String method;
-    private String webircPassword;
-	private String webircSpoofIP;
+    private final byte[] cloakKey; // 32 bytes of stuff to scramble the dest with
+    private final String hostname;
+    private final String method;
+    private final String webircPassword;
+    private final String webircSpoofIP;
 }

apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelServer.java

@@ -49,8 +49,8 @@ public class I2PTunnelServer extends I2PTunnelTask implements Runnable {
 
     protected Logging l;
 
-    private static final long DEFAULT_READ_TIMEOUT = -1; // 3*60*1000;
-    /** default timeout to 3 minutes - override if desired */
+    private static final long DEFAULT_READ_TIMEOUT = 5*60*1000;
+    /** default timeout to 5 minutes - override if desired */
     protected long readTimeout = DEFAULT_READ_TIMEOUT;
 
     /** do we use threads? default true (ignored for standard servers, always false) */
@@ -191,7 +191,7 @@ public class I2PTunnelServer extends I2PTunnelTask implements Runnable {
         if (_usePool) {
             String usePool = getTunnel().getClientOptions().getProperty(PROP_USE_POOL);
             if (usePool != null)
-                _usePool = Boolean.valueOf(usePool).booleanValue();
+                _usePool = Boolean.parseBoolean(usePool);
             else
                 _usePool = DEFAULT_USE_POOL;
         }
@@ -207,7 +207,7 @@ public class I2PTunnelServer extends I2PTunnelTask implements Runnable {
             if (sockMgr == null) {
                 // try to make this error sensible as it will happen...
                 String msg = "Unable to connect to the router at " + getTunnel().host + ':' + portNum +
-                             " and build tunnels for the server at " + getTunnel().listenHost + ':' + port;
+                             " and build tunnels for the server at " + host.getHostAddress() + ':' + port;
                 if (++retries < MAX_RETRIES) {
                     this.l.log(msg + ", retrying in " + (RETRY_DELAY / 1000) + " seconds");
                     _log.error(msg + ", retrying in " + (RETRY_DELAY / 1000) + " seconds");
@@ -223,7 +223,7 @@ public class I2PTunnelServer extends I2PTunnelTask implements Runnable {
 
         sockMgr.setName("Server");
         getTunnel().addSession(sockMgr.getSession());
-        l.log("Tunnels ready for server at " + getTunnel().listenHost + ':' + port);
+        l.log("Tunnels ready for server at " + host.getHostAddress() + ':' + port);
         notifyEvent("openServerResult", "ok");
         open = true;
     }

apps/i2ptunnel/java/src/net/i2p/i2ptunnel/TunnelController.java

@@ -19,8 +19,10 @@ import net.i2p.client.I2PSession;
 import net.i2p.data.Base32;
 import net.i2p.data.Destination;
 import net.i2p.i2ptunnel.socks.I2PSOCKSTunnel;
+import net.i2p.util.FileUtil;
 import net.i2p.util.I2PAppThread;
 import net.i2p.util.Log;
+import net.i2p.util.SecureFile;
 import net.i2p.util.SecureFileOutputStream;
 
 /**
@@ -43,6 +45,8 @@ public class TunnelController implements Logging {
     private boolean _running;
     private boolean _starting;
     
+    public static final String KEY_BACKUP_DIR = "i2ptunnel-keyBackup";
+
     /**
      * Create a new controller for a tunnel out of the specific config options.
      * The config may contain a large number of options - only ones that begin in
@@ -102,8 +106,17 @@ public class TunnelController implements Logging {
             Destination dest = client.createDestination(fos);
             String destStr = dest.toBase64();
             log("Private key created and saved in " + keyFile.getAbsolutePath());
+            log("You should backup this file in a secure place.");
             log("New destination: " + destStr);
-            log("Base32: " + Base32.encode(dest.calculateHash().getData()) + ".b32.i2p");
+            String b32 = Base32.encode(dest.calculateHash().getData()) + ".b32.i2p";
+            log("Base32: " + b32);
+            File backupDir = new SecureFile(I2PAppContext.getGlobalContext().getConfigDir(), KEY_BACKUP_DIR);
+            if (backupDir.isDirectory() || backupDir.mkdir()) {
+                String name = b32 + '-' + I2PAppContext.getGlobalContext().clock().now() + ".dat";
+                File backup = new File(backupDir, name);
+                if (FileUtil.copy(keyFile, backup, false, true))
+                    log("Private key backup saved to " + backup.getAbsolutePath());
+            }
         } catch (I2PException ie) {
             if (_log.shouldLog(Log.ERROR))
                 _log.error("Error creating new destination", ie);
@@ -307,7 +320,9 @@ public class TunnelController implements Logging {
                 I2PSession session = sessions.get(i);
                 if (_log.shouldLog(Log.INFO))
                     _log.info("Acquiring session " + session);
-                TunnelControllerGroup.getInstance().acquire(this, session);
+                TunnelControllerGroup group = TunnelControllerGroup.getInstance();
+                if (group != null)
+                    group.acquire(this, session);
             }
             _sessions = sessions;
         } else {
@@ -326,7 +341,9 @@ public class TunnelController implements Logging {
                 I2PSession s = _sessions.get(i);
                 if (_log.shouldLog(Log.INFO))
                     _log.info("Releasing session " + s);
-                TunnelControllerGroup.getInstance().release(this, s);
+                TunnelControllerGroup group = TunnelControllerGroup.getInstance();
+                if (group != null)
+                    group.release(this, s);
             }
             // _sessions.clear() ????
         } else {
@@ -536,8 +553,8 @@ public class TunnelController implements Logging {
     /** default true */
     public String getSharedClient() { return _config.getProperty("sharedClient", "true"); }
     /** default true */
-    public boolean getStartOnLoad() { return Boolean.valueOf(_config.getProperty("startOnLoad", "true")).booleanValue(); }
-    public boolean getPersistentClientKey() { return Boolean.valueOf(_config.getProperty("option.persistentClientKey")).booleanValue(); }
+    public boolean getStartOnLoad() { return Boolean.parseBoolean(_config.getProperty("startOnLoad", "true")); }
+    public boolean getPersistentClientKey() { return Boolean.parseBoolean(_config.getProperty("option.persistentClientKey")); }
 
     public String getMyDestination() {
         if (_tunnel != null) {

apps/i2ptunnel/java/src/net/i2p/i2ptunnel/TunnelControllerGroup.java

@@ -12,27 +12,35 @@ import java.util.Properties;
 import java.util.Set;
 
 import net.i2p.I2PAppContext;
+import net.i2p.app.*;
+import static net.i2p.app.ClientAppState.*;
 import net.i2p.client.I2PSession;
 import net.i2p.client.I2PSessionException;
 import net.i2p.data.DataHelper;
 import net.i2p.util.I2PAppThread;
 import net.i2p.util.Log;
 import net.i2p.util.OrderedProperties;
+import net.i2p.util.SystemVersion;
 
 /**
  * Coordinate a set of tunnels within the JVM, loading and storing their config
  * to disk, and building new ones as requested.
  *
- * Warning - this is a singleton. Todo: fix
+ * This is the entry point from clients.config.
  */
-public class TunnelControllerGroup {
-    private Log _log;
-    private static TunnelControllerGroup _instance;
+public class TunnelControllerGroup implements ClientApp {
+    private final Log _log;
+    private volatile ClientAppState _state;
+    private final I2PAppContext _context;
+    private final ClientAppManager _mgr;
+    private static volatile TunnelControllerGroup _instance;
     static final String DEFAULT_CONFIG_FILE = "i2ptunnel.config";
     
     private final List<TunnelController> _controllers;
-    private String _configFile = DEFAULT_CONFIG_FILE;
+    private final String _configFile;
     
+    private static final String REGISTERED_NAME = "i2ptunnel";
+
     /** 
      * Map of I2PSession to a Set of TunnelController objects 
      * using the session (to prevent closing the session until
@@ -41,48 +49,145 @@ public class TunnelControllerGroup {
      */
     private final Map<I2PSession, Set<TunnelController>> _sessions;
     
+    /**
+     *  In I2PAppContext will instantiate if necessary and always return non-null.
+     *  As of 0.9.4, when in RouterContext, will return null (except in Android)
+     *  if the TCG has not yet been started by the router.
+     *
+     *  @throws IllegalArgumentException if unable to load from i2ptunnel.config
+     */
     public static TunnelControllerGroup getInstance() { 
         synchronized (TunnelControllerGroup.class) {
-            if (_instance == null)
-                _instance = new TunnelControllerGroup(DEFAULT_CONFIG_FILE);
+            if (_instance == null) {
+                I2PAppContext ctx = I2PAppContext.getGlobalContext();
+                if (SystemVersion.isAndroid() || !ctx.isRouterContext()) {
+                    _instance = new TunnelControllerGroup(ctx, null, null);
+                    _instance.startup();
+                } // else wait for the router to start it
+            }
             return _instance; 
         }
     }
 
-    private TunnelControllerGroup(String configFile) {
-        _log = I2PAppContext.getGlobalContext().logManager().getLog(TunnelControllerGroup.class);
-        _controllers = Collections.synchronizedList(new ArrayList());
-        _configFile = configFile;
+    /**
+     *  Instantiation only. Caller must call startup().
+     *  Config file problems will not throw exception until startup().
+     *
+     *  @param mgr may be null
+     *  @param args one arg, the config file, if not absolute will be relative to the context's config dir,
+     *              if empty or null, the default is i2ptunnel.config
+     *  @since 0.9.4
+     */
+    public TunnelControllerGroup(I2PAppContext context, ClientAppManager mgr, String[] args) {
+        _state = UNINITIALIZED;
+        _context = context;
+        _mgr = mgr;
+        _log = _context.logManager().getLog(TunnelControllerGroup.class);
+        _controllers = new ArrayList();
+        if (args == null || args.length <= 0)
+            _configFile = DEFAULT_CONFIG_FILE;
+        else if (args.length == 1)
+            _configFile = args[0];
+        else
+            throw new IllegalArgumentException("Usage: TunnelControllerGroup [filename]");
         _sessions = new HashMap(4);
-        loadControllers(_configFile);
-        I2PAppContext.getGlobalContext().addShutdownTask(new Shutdown());
+        synchronized (TunnelControllerGroup.class) {
+            if (_instance == null)
+                _instance = this;
+        }
+        if (_instance != this) {
+            _log.logAlways(Log.WARN, "New TunnelControllerGroup, now you have two");
+            if (_log.shouldLog(Log.WARN))
+                _log.warn("I did it", new Exception());
+        }
+        _state = INITIALIZED;
     }
 
+    /**
+     *  @param args one arg, the config file, if not absolute will be relative to the context's config dir,
+     *              if no args, the default is i2ptunnel.config
+     *  @throws IllegalArgumentException if unable to load from config from file
+     */
     public static void main(String args[]) {
         synchronized (TunnelControllerGroup.class) {
             if (_instance != null) return; // already loaded through the web
-            
-            if ( (args == null) || (args.length <= 0) ) {
-                _instance = new TunnelControllerGroup(DEFAULT_CONFIG_FILE);
-            } else if (args.length == 1) {
-                _instance = new TunnelControllerGroup(args[0]);
-            } else {
-                System.err.println("Usage: TunnelControllerGroup [filename]");
-                return;
-            }
+            _instance = new TunnelControllerGroup(I2PAppContext.getGlobalContext(), null, args);
+            _instance.startup();
         }
     }
 
+    /**
+     *  ClientApp interface
+     *  @throws IllegalArgumentException if unable to load config from file
+     *  @since 0.9.4
+     */
+    public void startup() {
+        loadControllers(_configFile);
+        if (_mgr != null)
+            _mgr.register(this);
+            // RouterAppManager registers its own shutdown hook
+        else
+            _context.addShutdownTask(new Shutdown());
+    }
+
+    /**
+     *  ClientApp interface
+     *  @since 0.9.4
+     */
+    public ClientAppState getState() {
+        return _state;
+    }
+
+    /**
+     *  ClientApp interface
+     *  @since 0.9.4
+     */
+    public String getName() {
+        return REGISTERED_NAME;
+    }
+
+    /**
+     *  ClientApp interface
+     *  @since 0.9.4
+     */
+    public String getDisplayName() {
+        return REGISTERED_NAME;
+    }
+
+    /**
+     *  @since 0.9.4
+     */
+    private void changeState(ClientAppState state) {
+        changeState(state, null);
+    }
+
+    /**
+     *  @since 0.9.4
+     */
+    private synchronized void changeState(ClientAppState state, Exception e) {
+        _state = state;
+        if (_mgr != null)
+            _mgr.notify(this, state, null, e);
+    }
+
     /**
      *  Warning - destroys the singleton!
      *  @since 0.8.8
      */
-    private static class Shutdown implements Runnable {
+    private class Shutdown implements Runnable {
         public void run() {
             shutdown();
         }
     }
 
+    /**
+     *  ClientApp interface
+     *  @since 0.9.4
+     */
+    public void shutdown(String[] args) {
+        shutdown();
+    }
+
     /**
      *  Warning - destroys the singleton!
      *  Caller must root a new context before calling instance() or main() again.
@@ -91,28 +196,33 @@ public class TunnelControllerGroup {
      *
      *  @since 0.8.8
      */
-    public static void shutdown() {
+    public synchronized void shutdown() {
+        if (_state != STARTING && _state != RUNNING)
+            return;
+        changeState(STOPPING);
+        if (_mgr != null)
+            _mgr.unregister(this);
+        unloadControllers();
         synchronized (TunnelControllerGroup.class) {
-            if (_instance == null) return;
-            _instance.unloadControllers();
-            _instance._log = null;
-            _instance = null;
+            if (_instance == this)
+                _instance = null;
         }
+/// fixme static
         I2PTunnelClientBase.killClientExecutor();
+        changeState(STOPPED);
     }
     
     /**
      * Load up all of the tunnels configured in the given file (but do not start
      * them)
      *
+     * DEPRECATED for use outside this class. Use startup() or getInstance().
+     *
+     * @throws IllegalArgumentException if unable to load from file
      */
-    public void loadControllers(String configFile) {
+    public synchronized void loadControllers(String configFile) {
+        changeState(STARTING);
         Properties cfg = loadConfig(configFile);
-        if (cfg == null) {
-            if (_log.shouldLog(Log.WARN))
-                _log.warn("Unable to load the config from " + configFile);
-            return;
-        }
         int i = 0; 
         while (true) {
             String type = cfg.getProperty("tunnel." + i + ".type");
@@ -127,20 +237,28 @@ public class TunnelControllerGroup {
         
         if (_log.shouldLog(Log.INFO))
             _log.info(i + " controllers loaded from " + configFile);
+        changeState(RUNNING);
     }
     
     private class StartControllers implements Runnable {
         public void run() {
-            for (int i = 0; i < _controllers.size(); i++) {
-                TunnelController controller = _controllers.get(i);
-                if (controller.getStartOnLoad())
-                    controller.startTunnel();
+            synchronized(TunnelControllerGroup.this) {
+                for (int i = 0; i < _controllers.size(); i++) {
+                    TunnelController controller = _controllers.get(i);
+                    if (controller.getStartOnLoad())
+                        controller.startTunnel();
+                }
             }
         }
     }
     
-    
-    public void reloadControllers() {
+    /**
+     * Stop all tunnels, reload config, and restart those configured to do so.
+     * WARNING - Does NOT simply reload the configuration!!! This is probably not what you want.
+     *
+     * @throws IllegalArgumentException if unable to reload config file
+     */
+    public synchronized void reloadControllers() {
         unloadControllers();
         loadControllers(_configFile);
     }
@@ -150,7 +268,7 @@ public class TunnelControllerGroup {
      * file or do other silly things)
      *
      */
-    public void unloadControllers() {
+    public synchronized void unloadControllers() {
         stopAllControllers();
         _controllers.clear();
         if (_log.shouldLog(Log.INFO))
@@ -162,14 +280,14 @@ public class TunnelControllerGroup {
      * a config file or start it or anything)
      *
      */
-    public void addController(TunnelController controller) { _controllers.add(controller); }
+    public synchronized void addController(TunnelController controller) { _controllers.add(controller); }
     
     /**
      * Stop and remove the given tunnel
      *
      * @return list of messages from the controller as it is stopped
      */
-    public List<String> removeController(TunnelController controller) {
+    public synchronized List<String> removeController(TunnelController controller) {
         if (controller == null) return new ArrayList();
         controller.stopTunnel();
         List<String> msgs = controller.clearMessages();
@@ -183,7 +301,7 @@ public class TunnelControllerGroup {
      *
      * @return list of messages the tunnels generate when stopped
      */
-    public List<String> stopAllControllers() {
+    public synchronized List<String> stopAllControllers() {
         List<String> msgs = new ArrayList();
         for (int i = 0; i < _controllers.size(); i++) {
             TunnelController controller = _controllers.get(i);
@@ -200,7 +318,7 @@ public class TunnelControllerGroup {
      *
      * @return list of messages the tunnels generate when started
      */
-    public List<String> startAllControllers() {
+    public synchronized List<String> startAllControllers() {
         List<String> msgs = new ArrayList();
         for (int i = 0; i < _controllers.size(); i++) {
             TunnelController controller = _controllers.get(i);
@@ -218,7 +336,7 @@ public class TunnelControllerGroup {
      *
      * @return list of messages the tunnels generate when restarted
      */
-    public List<String> restartAllControllers() {
+    public synchronized List<String> restartAllControllers() {
         List<String> msgs = new ArrayList();
         for (int i = 0; i < _controllers.size(); i++) {
             TunnelController controller = _controllers.get(i);
@@ -235,7 +353,7 @@ public class TunnelControllerGroup {
      *
      * @return list of messages the tunnels have generated
      */
-    public List<String> clearAllMessages() {
+    public synchronized List<String> clearAllMessages() {
         List<String> msgs = new ArrayList();
         for (int i = 0; i < _controllers.size(); i++) {
             TunnelController controller = _controllers.get(i);
@@ -257,8 +375,7 @@ public class TunnelControllerGroup {
      * Save the configuration of all known tunnels to the given file
      *
      */
-    public void saveConfig(String configFile) throws IOException {
-        _configFile = configFile;
+    public synchronized void saveConfig(String configFile) throws IOException {
         File cfgFile = new File(configFile);
         if (!cfgFile.isAbsolute())
             cfgFile = new File(I2PAppContext.getGlobalContext().getConfigDir(), configFile);
@@ -279,16 +396,17 @@ public class TunnelControllerGroup {
     /**
      * Load up the config data from the file
      *
-     * @return properties loaded or null if there was an error
+     * @return properties loaded
+     * @throws IllegalArgumentException if unable to load from file
      */
-    private Properties loadConfig(String configFile) {
+    private synchronized Properties loadConfig(String configFile) {
         File cfgFile = new File(configFile);
         if (!cfgFile.isAbsolute())
             cfgFile = new File(I2PAppContext.getGlobalContext().getConfigDir(), configFile);
         if (!cfgFile.exists()) {
             if (_log.shouldLog(Log.ERROR))
                 _log.error("Unable to load the controllers from " + cfgFile.getAbsolutePath());
-            return null;
+            throw new IllegalArgumentException("Unable to load the controllers from " + cfgFile.getAbsolutePath());
         }
         
         Properties props = new Properties();
@@ -298,7 +416,7 @@ public class TunnelControllerGroup {
         } catch (IOException ioe) {
             if (_log.shouldLog(Log.ERROR))
                 _log.error("Error reading the controllers from " + cfgFile.getAbsolutePath(), ioe);
-            return null;
+            throw new IllegalArgumentException("Error reading the controllers from " + cfgFile.getAbsolutePath(), ioe);
         }
     }
     
@@ -307,7 +425,9 @@ public class TunnelControllerGroup {
      *
      * @return list of TunnelController objects
      */
-    public List<TunnelController> getControllers() { return _controllers; }
+    public synchronized List<TunnelController> getControllers() {
+        return new ArrayList(_controllers);
+    }
     
     
     /**