Pluck of 48ac112fe938c8e960413ad60f64d313a1e5c9ac and e415e24aca895c5a1f88380b1a3946e3b49749a9

* Add new *.i2p2.de certificate
* Added backup's ressed server. (Thanks backup!)
* Temporary removing h2ik's server since he's been AWOL since 06-03-2013 and he's reseed server is down.


applied changes from 185bc62d33224812060d2f68266a029935fb710d
             through 48ac112fe938c8e960413ad60f64d313a1e5c9ac

applied changes from 1194eb12084302380b4315b20e5ae0c574ce1be3
             through e415e24aca895c5a1f88380b1a3946e3b49749a9

.mtn-ignore

@@ -2,6 +2,7 @@
 # Use mtn add --no-respect-ignore foo.jar to ignore this ignore list
 _jsp\.java$
 \.bz2$
+\.tar$
 \.class$
 \.diff$
 \.exe$
@@ -15,12 +16,16 @@ _jsp\.java$
 \.su2$
 \.tar$
 \.war$
+.\deb$
 \.zip$
 ^\.
-^build/
+^build
 ^pkg-temp/
 ~$
-/build/
+/build
 /classes/
 ^debian/copyright
+^debian/changelog
 override.properties
+sloccount.sc
+^reports/

.tx/config

@@ -7,8 +7,12 @@ trans.da = apps/i2ptunnel/locale/messages_da.po
 trans.de = apps/i2ptunnel/locale/messages_de.po
 trans.es = apps/i2ptunnel/locale/messages_es.po
 trans.fr = apps/i2ptunnel/locale/messages_fr.po
+trans.hu = apps/i2ptunnel/locale/messages_hu.po
 trans.it = apps/i2ptunnel/locale/messages_it.po
+trans.nb = apps/i2ptunnel/locale/messages_nb.po
 trans.nl = apps/i2ptunnel/locale/messages_nl.po
+trans.pl = apps/i2ptunnel/locale/messages_pl.po
+trans.pt = apps/i2ptunnel/locale/messages_pt.po
 trans.ru = apps/i2ptunnel/locale/messages_ru.po
 trans.sv_SE = apps/i2ptunnel/locale/messages_sv.po
 trans.uk_UA = apps/i2ptunnel/locale/messages_uk.po
@@ -22,13 +26,17 @@ trans.ar = apps/routerconsole/locale/messages_ar.po
 trans.cs = apps/routerconsole/locale/messages_cs.po
 trans.da = apps/routerconsole/locale/messages_da.po
 trans.de = apps/routerconsole/locale/messages_de.po
-trans.et_EE = apps/routerconsole/locale/messages_ee.po
+trans.el = apps/routerconsole/locale/messages_el.po
 trans.es = apps/routerconsole/locale/messages_es.po
+trans.et_EE = apps/routerconsole/locale/messages_et.po
 trans.fi = apps/routerconsole/locale/messages_fi.po
 trans.fr = apps/routerconsole/locale/messages_fr.po
+trans.hu = apps/routerconsole/locale/messages_hu.po
 trans.it = apps/routerconsole/locale/messages_it.po
+trans.nb = apps/routerconsole/locale/messages_nb.po
 trans.nl = apps/routerconsole/locale/messages_nl.po
 trans.pl = apps/routerconsole/locale/messages_pl.po
+trans.pt = apps/routerconsole/locale/messages_pt.po
 trans.ru = apps/routerconsole/locale/messages_ru.po
 trans.sv_SE = apps/routerconsole/locale/messages_sv.po
 trans.uk_UA = apps/routerconsole/locale/messages_uk.po
@@ -43,7 +51,9 @@ trans.cs = apps/i2psnark/locale/messages_cs.po
 trans.de = apps/i2psnark/locale/messages_de.po
 trans.es = apps/i2psnark/locale/messages_es.po
 trans.fr = apps/i2psnark/locale/messages_fr.po
+trans.hu = apps/i2psnark/locale/messages_hu.po
 trans.it = apps/i2psnark/locale/messages_it.po
+trans.nb = apps/i2psnark/locale/messages_nb.po
 trans.nl = apps/i2psnark/locale/messages_nl.po
 trans.pl = apps/i2psnark/locale/messages_pl.po
 trans.pt = apps/i2psnark/locale/messages_pt.po
@@ -59,11 +69,14 @@ trans.ar = apps/susidns/locale/messages_ar.po
 trans.cs = apps/susidns/locale/messages_cs.po
 trans.da = apps/susidns/locale/messages_da.po
 trans.de = apps/susidns/locale/messages_de.po
+trans.el = apps/susidns/locale/messages_el.po
 trans.es = apps/susidns/locale/messages_es.po
 trans.fr = apps/susidns/locale/messages_fr.po
+trans.hu = apps/susidns/locale/messages_hu.po
 trans.it = apps/susidns/locale/messages_it.po
 trans.nl = apps/susidns/locale/messages_nl.po
 trans.pl = apps/susidns/locale/messages_pl.po
+trans.pt = apps/susidns/locale/messages_pt.po
 trans.ru = apps/susidns/locale/messages_ru.po
 trans.sv_SE = apps/susidns/locale/messages_sv.po
 trans.uk_UA = apps/susidns/locale/messages_uk.po
@@ -77,8 +90,10 @@ trans.ar = apps/desktopgui/locale/messages_ar.po
 trans.cs = apps/desktopgui/locale/messages_cs.po
 trans.da = apps/desktopgui/locale/messages_da.po
 trans.de = apps/desktopgui/locale/messages_de.po
+trans.el = apps/desktopgui/locale/messages_el.po
 trans.es = apps/desktopgui/locale/messages_es.po
 trans.fr = apps/desktopgui/locale/messages_fr.po
+trans.hu = apps/desktopgui/locale/messages_hu.po
 trans.it = apps/desktopgui/locale/messages_it.po
 trans.nl = apps/desktopgui/locale/messages_nl.po
 trans.pl = apps/desktopgui/locale/messages_pl.po
@@ -95,6 +110,7 @@ trans.cs = apps/susimail/locale/messages_cs.po
 trans.de = apps/susimail/locale/messages_de.po
 trans.es = apps/susimail/locale/messages_es.po
 trans.fr = apps/susimail/locale/messages_fr.po
+trans.hu = apps/susimail/locale/messages_hu.po
 trans.it = apps/susimail/locale/messages_it.po
 trans.nl = apps/susimail/locale/messages_nl.po
 trans.ru = apps/susimail/locale/messages_ru.po
@@ -109,12 +125,23 @@ source_file = debian/po/templates.pot
 source_lang = en
 trans.cs = debian/po/cs.po
 trans.de = debian/po/de.po
+trans.el = debian/po/el.po
 trans.es = debian/po/es.po
+trans.fr = debian/po/fr.po
+trans.it = debian/po/it.po
+trans.hu = debian/po/hu.po
 trans.pl = debian/po/pl.po
 trans.ru = debian/po/ru.po
 trans.sv_SE = debian/po/sv.po
 trans.uk_UA = debian/po/uk.po
 
+[I2P.i2prouter-script]
+source_file = installer/resources/locale/messages_en.po
+source_lang = en
+trans.de = installer/resources/locale/po/messages_de.po
+trans.fr = installer/resources/locale/po/messages_fr.po
+trans.it = installer/resources/locale/po/messages_it.po
+
 [main]
 host = http://www.transifex.net
 

INSTALL-headless.txt

@@ -11,7 +11,7 @@ you may use:
   lynx http://localhost:7657/
 to configure the router.
 
-If you're having trouble, swing by http://forum.i2p2.de/, check the
+If you're having trouble, swing by http://forum.i2p/, check the
 website at http://www.i2p2.de/, or get on irc://irc.freenode.net/#i2p
 
 I2P will create and store files and configuration data in the user directory

LICENSE.txt

@@ -72,6 +72,9 @@ Public domain except as listed below:
    Copyright (c) 2006, Matthew Estes
    See licenses/LICENSE-BlockFile.txt
 
+   SipHashInline:
+   Copyright 2012  Hiroshi Nakamura <nahi@ruby-lang.org>
+   See licenses/LICENSE-Apache2.0.txt
 
 Router (router.jar):
 Public domain except as listed below:
@@ -79,11 +82,13 @@ Public domain except as listed below:
    From freenet
    See licenses/LICENSE-GPLv2.txt
 
-   UPnP subsystem 1.7:
-   Copyright (C) 2003-2006 Satoshi Konno
+   UPnP subsystem (CyberLink) 2.1:
+   Copyright (C) 2003-2010 Satoshi Konno
    See licenses/LICENSE-UPnP.txt
 
-   GeoIP data free to use, courtesy http://www.maxmind.com/
+   GeoIP: GeoLite databases are licensed under a Creative Commons Attribution-ShareAlike 3.0 Unported License.
+   http://creativecommons.org/licenses/by-sa/3.0/
+   This product includes GeoLite data created by MaxMind, available from http://www.maxmind.com/
 
 
 Installer:
@@ -172,8 +177,9 @@ Applications:
    By welterde.
    See licenses/LICENSE-GPLv2.txt
 
-   Jetty 5.1.15:
-   Copyright 2000-2004 Mort Bay Consulting Pty. Ltd.
+   Jetty 6.1.26:
+   Copyright 1995-2009 Mort Bay Consulting Pty Ltd
+   See licenses/LICENSE-Jetty.txt
    See licenses/LICENSE-Apache2.0.txt
    See licenses/NOTICE-Commons-Logging.txt
 
@@ -194,8 +200,10 @@ Applications:
          - Jersey and EU flag icons: public domain, courtesy Xrmap flag
            collection http://www.arvernes.com/wiki/index.php/Xrmap
          - Guernsey and Isle of Man flags from the Open Clip Art Library, released into the public domain
+         - Curaçao, courtesy of David Benbennick, released into the public domain
          - All other flag icons: public domain, courtesy mjames@gmail.com http://www.famfamfam.com/
        Silk icons: See licenses/LICENSE-SilkIcons.txt
+       FatCow icons: See licenses/LICENSE-FatCowIcons.txt
 
        GeoIP Data:
        Copyright (c) 2008 MaxMind, Inc.  All Rights Reserved.
@@ -205,6 +213,10 @@ Applications:
    "Man with hat over face" & related images licensed under a Creative Commons 2.0 license.
    Original photos by Florian Kuhlmann. http://www.flickr.com/photos/floriankuhlmann/3117758155
 
+   I2PSnark light theme:
+   "Creative Commons Cat" licensed under a Creative Commons Attribution 3.0 Unported License.
+   Original photo by Boaz Arad. http://www.luxphile.com/2011/01/creative-commons-cat.html
+
    SAM:
    Public domain.
 
@@ -215,8 +227,10 @@ Applications:
    Copyright (C) 2005 <susi23@mail.i2p>
    GPLv2 (or any later version)
    See licenses/LICENSE-GPLv2.txt
-       Uses Apache Jakarta Standard Tag Library 1.1.2:
-       See licenses/LICENSE-Apache2.0.txt
+       Uses Glassfish Standard Tag Library (JSTL) 1.2:
+       Common Development and Distribution License (CDDL) version 1.0 + GNU General Public License (GPL) version 2
+       See https://glassfish.dev.java.net/public/CDDL+GPL.html
+       See licenses/LICENSE-GPLv2.txt
 
    SusiMail:
    Copyright (C) 2004-2005  <susi23@mail.i2p>
@@ -228,6 +242,10 @@ Applications:
       Bundles systray4j-2.4.1:
       See licenses/LICENSE-LGPLv2.1.txt
 
+   Tomcat 6.0.36:
+   Copyright 1999-2012 The Apache Software Foundation
+   See licenses/LICENSE-Apache2.0.txt
+   See licenses/NOTICE-Tomcat.txt
 
 
 Other Applications and Libraries

README.txt

@@ -32,7 +32,7 @@ FAQ:
 
 Need help?
 	IRC irc.freenode.net #i2p
-	http://forum.i2p2.de/
+	http://forum.i2p/
 
 Licenses:
 	See LICENSE.txt

Slackware/i2p/doinst.sh

@@ -26,9 +26,9 @@ for i in *.config ; {
 
 ( cd $INST_DIR/eepsite
 	if [ -f jetty.xml ] ; then
-		rm jetty.xml.new
+		echo "Please check ${INST_DIR}/eepsite, as there are new files."
 	else
-		mv jetty.xml.new jetty.xml
+		find $PKG/$INSTALL_DIR/i2p -name "*.xml.new" -exec sh -c 'mv "$0" "${0/.new}"' {} \;
 	fi
 )
 

Slackware/i2p/i2p.SlackBuild

@@ -18,6 +18,7 @@
 #
 
 BUILD=1sponge
+# INSTALL_DIR is referenced from /, don't prefix it with a '/'
 INSTALL_DIR=opt
 NAME=i2p
 ARCH=noarch
@@ -104,7 +105,10 @@ sed "s|%INSTALL_PATH|/$INSTALL_DIR/i2p|g" runplain.sh > a
 sed "s|%SYSTEM_java_io_tmpdir|/$INSTALL_DIR/i2p|g" a > runplain.sh
 # i2prouter %INSTALL_PATH and %SYSTEM_java_io_tmpdir
 sed "s|%INSTALL_PATH|/$INSTALL_DIR/i2p|g" i2prouter > a
-sed "s|%SYSTEM_java_io_tmpdir|/$INSTALL_DIR/i2p|g" a > i2prouter
+rm i2prouter
+mv a i2prouter
+sed "s|%SYSTEM_java_io_tmpdir|/$INSTALL_DIR/i2p|g" i2prouter > a
+sed "s|#ALLOW_ROOT=true|ALLOW_ROOT=true|g" a > i2prouter
 
 chmod 744 ./i2prouter
 chmod 744 ./osid
@@ -116,7 +120,7 @@ rm -Rf ./lib/*.dll ./*.bat ./*.exe ./installer ./icons ./a postinstall.sh
 
 mv $PKG/$INSTALL_DIR/i2p/*.config $PKG/install
 mv $PKG/$INSTALL_DIR/i2p/blocklist.txt $PKG/$INSTALL_DIR/i2p/blocklist.txt.new
-mv $PKG/$INSTALL_DIR/i2p/eepsite/jetty.xml $PKG/$INSTALL_DIR/i2p/eepsite/jetty.xml.new
+find $PKG/$INSTALL_DIR/i2p -name "*.xml" -exec mv {} {}.new \;
 mv $PKG/$INSTALL_DIR/i2p/eepsite/docroot/index.html $PKG/$INSTALL_DIR/i2p/eepsite/docroot/index.html.new
 mv $PKG/$INSTALL_DIR/i2p/eepsite/docroot/favicon.ico $PKG/$INSTALL_DIR/i2p/eepsite/docroot/favicon.ico.new
 sed "s|directory|/$INSTALL_DIR/i2p/|g" $CWD/doinst.sh > $PKG/install/doinst.sh
@@ -124,7 +128,7 @@ cat $CWD/slack-desc > $PKG/install/slack-desc
 
 cd $PKG
 #
-# requiredbuilder fucks up REALLY bad, and thinks java is perl?!
+# requiredbuilder messes up REALLY bad, and thinks java is perl?!
 # It also did not catch the shell requirements! BOOOOOOOOOOO! HISSSSSSSS!
 #
 # requiredbuilder -v -y -s $CWD $PKG

apps/BOB/Demos/echo/echoclient/src/net/i2p/BOB/Demos/echo/echoclient/Main.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB.Demos.echo.echoclient;
 
@@ -55,7 +47,7 @@ public class Main {
 				// exit on anything not legal
 				break;
 			}
-			c = (char)(b & 0x7f); // We only really give a fuck about ASCII
+			c = (char)(b & 0x7f); // We only care about ASCII
 			S = new String(S + c);
 		}
 		return S;

apps/BOB/Demos/echo/echoserver/src/net/i2p/BOB/Demos/echo/echoserver/Main.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB.Demos.echo.echoserver;
 
@@ -52,7 +44,7 @@ public class Main {
 			if(b < 20) {
 				break;
 			}
-			c = (char)(b & 0x7f); // We only really give a fuck about ASCII
+			c = (char)(b & 0x7f); // We only care about ASCII
 			S = new String(S + c);
 		}
 		return S;

apps/BOB/build.xml

@@ -83,7 +83,7 @@
 	        <pathelement path="${javac.classpath}" />
 	    </path>
 	</copy>
-	<copy todir="${dist.dir}/lib" file="../../installer/lib/jbigi/jbigi.jar" />
+	<copy todir="${dist.dir}/lib" file="../../build/jbigi.jar" />
 
 	<!-- Extract the classes inside the jar files -->
 	<unjar dest="${dist.dir}/classes" >

apps/BOB/nbproject/project.properties

@@ -30,11 +30,11 @@ excludes=**/*.html,**/*.txt
 file.reference.build-javadoc=../../i2p.i2p/build/javadoc
 file.reference.i2p.jar=../../core/java/build/i2p.jar
 file.reference.i2ptunnel.jar=../i2ptunnel/java/build/i2ptunnel.jar
-file.reference.jbigi.jar=../../installer/lib/jbigi/jbigi.jar
+file.reference.jbigi.jar=../../build/jbigi.jar
 file.reference.mstreaming.jar=../ministreaming/java/build/mstreaming.jar
 file.reference.router.jar=../../router/java/build/router.jar
 file.reference.streaming.jar=../streaming/java/build/streaming.jar
-file.reference.wrapper.jar=../../installer/lib/wrapper/linux/wrapper.jar
+file.reference.wrapper.jar=../../installer/lib/wrapper/all/wrapper.jar
 includes=**
 jar.compress=true
 javac.classpath=\

apps/BOB/src/net/i2p/BOB/BOB.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB;
 
@@ -38,7 +30,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
 
 import net.i2p.I2PAppContext;
 import net.i2p.client.I2PClient;
-import net.i2p.client.streaming.RetransmissionTimer;
 import net.i2p.util.Log;
 import net.i2p.util.SimpleScheduler;
 import net.i2p.util.SimpleTimer2;
@@ -116,7 +107,6 @@ import net.i2p.util.SimpleTimer2;
  */
 public class BOB {
 
-	private final static Log _log = new Log(BOB.class);
 	public final static String PROP_CONFIG_LOCATION = "BOB.config";
 	public final static String PROP_BOB_PORT = "BOB.port";
 	public final static String PROP_BOB_HOST = "BOB.host";
@@ -138,7 +128,7 @@ public class BOB {
 	 */
 	public static void info(String arg) {
 		System.out.println("INFO:" + arg);
-		_log.info(arg);
+		(new Log(BOB.class)).info(arg);
 	}
 
 	/**
@@ -148,7 +138,7 @@ public class BOB {
 	 */
 	public static void warn(String arg) {
 		System.out.println("WARNING:" + arg);
-		_log.warn(arg);
+		(new Log(BOB.class)).warn(arg);
 	}
 
 	/**
@@ -158,7 +148,7 @@ public class BOB {
 	 */
 	public static void error(String arg) {
 		System.out.println("ERROR: " + arg);
-		_log.error(arg);
+		(new Log(BOB.class)).error(arg);
 	}
 
 	/**
@@ -182,12 +172,11 @@ public class BOB {
 		// Re-reading the config file in each thread is pretty damn stupid.
 		String configLocation = System.getProperty(PROP_CONFIG_LOCATION, "bob.config");
 		// This is here just to ensure there is no interference with our threadgroups.
-		RetransmissionTimer Y = RetransmissionTimer.getInstance();
 		SimpleScheduler Y1 = SimpleScheduler.getInstance();
 		SimpleTimer2 Y2 = SimpleTimer2.getInstance();
-		i = Y.hashCode();
 		i = Y1.hashCode();
 		i = Y2.hashCode();
+		Log _log = new Log(BOB.class);
 		try {
 			{
 				File cfg = new File(configLocation);
@@ -263,6 +252,7 @@ public class BOB {
 
 			i = 0;
 			boolean g = false;
+			spin.set(true);
 			try {
 				info("BOB is now running.");
 				listener = new ServerSocket(Integer.parseInt(props.getProperty(PROP_BOB_PORT)), 10, InetAddress.getByName(props.getProperty(PROP_BOB_HOST)));

apps/BOB/src/net/i2p/BOB/DoCMDS.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB;
 
@@ -54,7 +46,7 @@ public class DoCMDS implements Runnable {
 
 	// FIX ME
 	// I need a better way to do versioning, but this will do for now.
-	public static final String BMAJ = "00",  BMIN = "00",  BREV = "0F",  BEXT = "";
+	public static final String BMAJ = "00",  BMIN = "00",  BREV = "10",  BEXT = "";
 	public static final String BOBversion = BMAJ + "." + BMIN + "." + BREV + BEXT;
 	private Socket server;
 	private Properties props;
@@ -400,7 +392,7 @@ public class DoCMDS implements Runnable {
 	 */
 	private boolean is64ok(String data) {
 		try {
-			Destination x = new Destination(data);
+			new Destination(data);
 			return true;
 		} catch (Exception e) {
 			return false;

apps/BOB/src/net/i2p/BOB/I2Plistener.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB;
 
@@ -41,7 +33,6 @@ public class I2Plistener implements Runnable {
 
 	private NamedDB info,  database;
 	private Log _log;
-//	private int tgwatch;
 	public I2PSocketManager socketManager;
 	public I2PServerSocket serverSocket;
 	private AtomicBoolean lives;
@@ -95,7 +86,7 @@ public class I2Plistener implements Runnable {
 
 				}
 			} catch (I2PException e) {
-				// bad shit
+				// bad stuff
 				System.out.println("Exception " + e);
 			}
 		} finally {
@@ -103,7 +94,7 @@ public class I2Plistener implements Runnable {
 				serverSocket.close();
 			} catch (I2PException ex) {
 			}
-		// System.out.println("I2Plistener: Close");
+			// System.out.println("I2Plistener: Close");
 		}
 	}
 }

apps/BOB/src/net/i2p/BOB/I2PtoTCP.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB;
 

apps/BOB/src/net/i2p/BOB/Lifted.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB;
 

apps/BOB/src/net/i2p/BOB/MUXlisten.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB;
 
@@ -108,16 +100,6 @@ public class MUXlisten implements Runnable {
 				this.listener = new ServerSocket(port, backlog, host);
 			}
 			socketManager = I2PSocketManagerFactory.createManager(prikey, Q);
-		// I2PException, IOException, RuntimeException
-		// To bad we can't just catch and enumerate....
-		// } catch (I2PException e) {
-		// Something went bad.
-		//	this.database.getWriteLock();
-		//	this.info.getWriteLock();
-		//	this.info.add("STARTING", new Boolean(false));
-		//	this.info.releaseWriteLock();
-		//	this.database.releaseWriteLock();
-		//	throw new I2PException(e);
 		} catch (IOException e) {
 			// Something went bad.
 			this.database.getWriteLock();
@@ -194,7 +176,6 @@ public class MUXlisten implements Runnable {
 				lock.set(false);
 				return;
 			}
-//		socketManager.addDisconnectListener(new DisconnectListener());
 			lives.set(true);
 			lock.set(false);
 			quit:
@@ -313,14 +294,14 @@ public class MUXlisten implements Runnable {
 
 			// Hopefully nuke stuff here...
 			{
-				String boner = tg.getName();
+				String groupName = tg.getName();
 				try {
-					_log.warn("destroySocketManager " + boner);
+					_log.warn("destroySocketManager " + groupName);
 					socketManager.destroySocketManager();
-					_log.warn("destroySocketManager Successful" + boner);
+					_log.warn("destroySocketManager Successful" + groupName);
 				} catch (Exception e) {
 					// nop
-					_log.warn("destroySocketManager Failed" + boner);
+					_log.warn("destroySocketManager Failed" + groupName);
 					_log.warn(e.toString());
 				}
 			}
@@ -344,26 +325,25 @@ public class MUXlisten implements Runnable {
 
 			// Wait around till all threads are collected.
 			if (tg != null) {
-				String boner = tg.getName();
-				// System.out.println("BOB: MUXlisten: Starting thread collection for: " + boner);
-				_log.warn("BOB: MUXlisten: Starting thread collection for: " + boner);
-				// tg.interrupt(); // give my stuff a small smack again.
+				String groupName = tg.getName();
+				// System.out.println("BOB: MUXlisten: Starting thread collection for: " + groupName);
+				_log.warn("BOB: MUXlisten: Starting thread collection for: " + groupName);
 				if (tg.activeCount() + tg.activeGroupCount() != 0) {
-					// visit(tg, 0, boner);
+					// visit(tg, 0, groupName);
 					int foo = tg.activeCount() + tg.activeGroupCount();
 					// hopefully no longer needed!
 					// int bar = lives;
-					// System.out.println("BOB: MUXlisten: Waiting on threads for " + boner);
-					// System.out.println("\nBOB: MUXlisten: ThreadGroup dump BEGIN " + boner);
-					// visit(tg, 0, boner);
-					// System.out.println("BOB: MUXlisten: ThreadGroup dump END " + boner + "\n");
+					// System.out.println("BOB: MUXlisten: Waiting on threads for " + groupName);
+					// System.out.println("\nBOB: MUXlisten: ThreadGroup dump BEGIN " + groupName);
+					// visit(tg, 0, groupName);
+					// System.out.println("BOB: MUXlisten: ThreadGroup dump END " + groupName + "\n");
 					// Happily spin forever :-(
 					while (foo != 0) {
 						foo = tg.activeCount() + tg.activeGroupCount();
 						// if (lives != bar && lives != 0) {
-						// 	System.out.println("\nBOB: MUXlisten: ThreadGroup dump BEGIN " + boner);
-						// 	visit(tg, 0, boner);
-						// 	System.out.println("BOB: MUXlisten: ThreadGroup dump END " + boner + "\n");
+						// 	System.out.println("\nBOB: MUXlisten: ThreadGroup dump BEGIN " + groupName);
+						// 	visit(tg, 0, groupName);
+						// 	System.out.println("BOB: MUXlisten: ThreadGroup dump END " + groupName + "\n");
 						// }
 						// bar = lives;
 						try {
@@ -373,8 +353,8 @@ public class MUXlisten implements Runnable {
 						}
 					}
 				}
-				// System.out.println("BOB: MUXlisten: Threads went away. Success: " + boner);
-				_log.warn("BOB: MUXlisten: Threads went away. Success: " + boner);
+				// System.out.println("BOB: MUXlisten: Threads went away. Success: " + groupName);
+				_log.warn("BOB: MUXlisten: Threads went away. Success: " + groupName);
 				tg.destroy();
 				// Zap reference to the ThreadGroup so the JVM can GC it.
 				tg = null;

apps/BOB/src/net/i2p/BOB/Main.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,11 +11,10 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB;
 
-import net.i2p.client.streaming.RetransmissionTimer;
 import net.i2p.util.SimpleScheduler;
 import net.i2p.util.SimpleTimer2;
 
@@ -40,7 +31,6 @@ public class Main {
 	 */
 	public static void main(String[] args) {
 		// THINK THINK THINK THINK THINK THINK
-		RetransmissionTimer Y = RetransmissionTimer.getInstance();
 		SimpleScheduler Y1 = SimpleScheduler.getInstance();
 		SimpleTimer2 Y2 = SimpleTimer2.getInstance();
 
@@ -48,6 +38,5 @@ public class Main {
 
 		Y2.stop();
 		Y1.stop();
-		Y.stop();
 	}
 }

apps/BOB/src/net/i2p/BOB/NamedDB.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB;
 

apps/BOB/src/net/i2p/BOB/TCPio.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB;
 
@@ -95,14 +87,9 @@ public class TCPio implements Runnable {
 					if (b > 0) {
 						Aout.write(a, 0, b);
 					} else if (b == 0) {
-// Will this die? We'll see.
 						while(Ain.available() == 0) {
 							Thread.sleep(20);
 						}
-//						Thread.yield(); // this should act like a mini sleep.
-//						if (Ain.available() == 0) {
-//							Thread.sleep(10);
-//						}
 					} else {
 						/* according to the specs:
 						 *

apps/BOB/src/net/i2p/BOB/TCPlistener.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB;
 
@@ -27,8 +19,6 @@ import java.io.IOException;
 import java.net.ServerSocket;
 import java.net.Socket;
 import java.net.SocketTimeoutException;
-// import net.i2p.client.I2PSession;
-// import net.i2p.client.I2PSessionException;
 import java.util.concurrent.atomic.AtomicBoolean;
 import net.i2p.client.streaming.I2PServerSocket;
 import net.i2p.client.streaming.I2PSocketManager;

apps/BOB/src/net/i2p/BOB/TCPtoI2P.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB;
 
@@ -94,7 +86,7 @@ public class TCPtoI2P implements Runnable {
 				// exit on anything not legal
 				break;
 			}
-			c = (char) (b & 0x7f); // We only really give a fuck about ASCII
+			c = (char) (b & 0x7f); // We only care about ASCII
 			S = new String(S + c);
 		}
 		return S;
@@ -108,20 +100,18 @@ public class TCPtoI2P implements Runnable {
 	 * @throws java.io.IOException
 	 */
 	private void Emsg(String e, OutputStream out) throws IOException {
-// Debugging		System.out.println("ERROR TCPtoI2P: " + e);
+		// Debugging		System.out.println("ERROR TCPtoI2P: " + e);
 		out.write("ERROR ".concat(e).getBytes());
 		out.write(13);
 		out.write(10);
 		out.flush();
 	}
 
-//	private void rlock() throws Exception {
 	private void rlock() {
 		database.getReadLock();
 		info.getReadLock();
 	}
 
-//	private void runlock() throws Exception {
 	private void runlock() {
 		info.releaseReadLock();
 		database.releaseReadLock();

apps/BOB/src/net/i2p/BOB/UDPIOthread.java

@@ -1,17 +1,9 @@
 /**
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ *                    WTFPL
  *                    Version 2, December 2004
  *
  * Copyright (C) sponge
  *   Planet Earth
- * Everyone is permitted to copy and distribute verbatim or modified
- * copies of this license document, and changing it is allowed as long
- * as the name is changed.
- *
- *            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
- *   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
- *
- *  0. You just DO WHAT THE FUCK YOU WANT TO.
  *
  * See...
  *
@@ -19,7 +11,7 @@
  *	and
  *	http://en.wikipedia.org/wiki/WTFPL
  *
- * ...for any additional details and liscense questions.
+ * ...for any additional details and license questions.
  */
 package net.i2p.BOB;
 
@@ -111,7 +103,7 @@ public class UDPIOthread implements I2PSessionListener, Runnable {
 	 * @param size
 	 */
 	public void messageAvailable(I2PSession session, int msgId, long size) {
-//		_log.debug("Message available: id = " + msgId + " size = " + size);
+		//		_log.debug("Message available: id = " + msgId + " size = " + size);
 		try {
 			byte msg[] = session.receiveMessage(msgId);
 			out.write(msg);

apps/addressbook/build.xml

@@ -56,6 +56,7 @@
 			<manifest>
 				<attribute name="Main-Class" value="addressbook.Daemon"/>
 				<attribute name="Implementation-Version" value="${full.version}" />
+				<attribute name="Built-By" value="${build.built-by}" />
 		                <attribute name="Build-Date" value="${build.timestamp}" />
 		                <attribute name="Base-Revision" value="${workspace.version}" />
 		                <attribute name="Workspace-Changes" value="${workspace.changes}" />
@@ -75,6 +76,7 @@
 		<war basedir="${dist}/tmp" webxml="web.xml" destfile="${dist}/${war}">
 			<manifest>
 				<attribute name="Implementation-Version" value="${full.version}" />
+				<attribute name="Built-By" value="${build.built-by}" />
 		                <attribute name="Build-Date" value="${build.timestamp}" />
 		                <attribute name="Base-Revision" value="${workspace.version}" />
 		                <attribute name="Workspace-Changes" value="${workspace.changes.tr}" />

apps/addressbook/java/src/net/i2p/addressbook/ConfigParser.java

@@ -37,6 +37,7 @@ import java.util.Map;
 
 import net.i2p.util.SecureFile;
 import net.i2p.util.SecureFileOutputStream;
+import net.i2p.util.SystemVersion;
 
 /**
  * Utility class providing methods to parse and write files in config file
@@ -49,7 +50,7 @@ import net.i2p.util.SecureFileOutputStream;
  */
 class ConfigParser {
 
-    private static final boolean isWindows = System.getProperty("os.name").startsWith("Win");
+    private static final boolean isWindows = SystemVersion.isWindows();
 
     /**
      * Strip the comments from a String. Lines that begin with '#' and ';' are

apps/addressbook/java/src/net/i2p/addressbook/Daemon.java

@@ -230,7 +230,7 @@ public class Daemon {
      */
     public static void update(Map<String, String> settings, String home) {
         File published = null;
-        boolean should_publish = Boolean.valueOf(settings.get("should_publish")).booleanValue();
+        boolean should_publish = Boolean.parseBoolean(settings.get("should_publish"));
         if (should_publish) 
             published = new File(home, settings
                 .get("published_addressbook"));
@@ -261,7 +261,7 @@ public class Daemon {
 
         // If false, add hosts via naming service; if true, write hosts.txt file directly
         // Default false
-        if (Boolean.valueOf(settings.get("update_direct")).booleanValue()) {
+        if (Boolean.parseBoolean(settings.get("update_direct"))) {
             // Direct hosts.txt access
             File routerFile = new File(home, settings.get("router_addressbook"));
             AddressBook master;

apps/addressbook/java/src/net/i2p/addressbook/Servlet.java

@@ -42,8 +42,8 @@ import javax.servlet.http.HttpServletResponse;
  */
 public class Servlet extends HttpServlet {
     private DaemonThread thread;
-    private String nonce;
-    private static final String PROP_NONCE = "addressbook.nonce";
+    //private String nonce;
+    //private static final String PROP_NONCE = "addressbook.nonce";
 
     /**
      * Hack to allow susidns to kick the daemon when the subscription list changes.
@@ -54,15 +54,15 @@ public class Servlet extends HttpServlet {
      */
     public void service(HttpServletRequest request, HttpServletResponse response) throws IOException {
         //System.err.println("Got request nonce = " + request.getParameter("nonce"));
-        if (this.thread != null && request.getParameter("wakeup") != null &&
-            this.nonce != null && this.nonce.equals(request.getParameter("nonce"))) {
-            //System.err.println("Sending interrupt");
-            this.thread.interrupt();
-            // no output
-        } else {
+        //if (this.thread != null && request.getParameter("wakeup") != null &&
+        //    this.nonce != null && this.nonce.equals(request.getParameter("nonce"))) {
+        //    //System.err.println("Sending interrupt");
+        //    this.thread.interrupt();
+        //    // no output
+        //} else {
             PrintWriter out = response.getWriter();
             out.write("I2P addressbook OK");
-        }
+        //}
     }
 
     /* (non-Javadoc)
@@ -75,16 +75,16 @@ public class Servlet extends HttpServlet {
         } catch (ServletException exp) {
             System.err.println("Addressbook init exception: " + exp);
         }
-        this.nonce = "" + Math.abs((new Random()).nextLong());
+        //this.nonce = "" + Math.abs((new Random()).nextLong());
         // put the nonce where susidns can get it
-        System.setProperty(PROP_NONCE, this.nonce);
+        //System.setProperty(PROP_NONCE, this.nonce);
         String[] args = new String[1];
         args[0] = config.getInitParameter("home");
         this.thread = new DaemonThread(args);
         this.thread.setDaemon(true);
         this.thread.setName("Addressbook");
         this.thread.start();
-        System.out.println("INFO: Starting Addressbook " + Daemon.VERSION);
+        //System.out.println("INFO: Starting Addressbook " + Daemon.VERSION);
         //System.out.println("INFO: config root under " + args[0]);
     }
 

apps/addressbook/java/src/net/i2p/addressbook/SubscriptionIterator.java

@@ -26,7 +26,6 @@ import java.util.Iterator;
 import java.util.List;
 
 import net.i2p.I2PAppContext;
-import net.i2p.data.DataHelper;  // debug
 
 /**
  * An iterator over the subscriptions in a SubscriptionList.  Note that this iterator

apps/desktopgui/.classpath

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry kind="src" path="src"/>
+	<classpathentry combineaccessrules="false" kind="src" path="/i2p_router"/>
+	<classpathentry combineaccessrules="false" kind="src" path="/i2p_sdk"/>
+	<classpathentry kind="lib" path="/lib/wrapper/all/wrapper.jar"/>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+	<classpathentry kind="output" path="build"/>
+</classpath>

apps/desktopgui/.project

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>desktopgui</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.jdt.core.javabuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.jdt.core.javanature</nature>
+	</natures>
+</projectDescription>

apps/desktopgui/build.xml

@@ -8,12 +8,15 @@
 	<property name="resources" value="resources"/>
     <property name="javadoc" value="javadoc"/>
 
+    <condition property="no.bundle">
+        <isfalse value="${require.gettext}" />
+    </condition>
 	<property name="javac.compilerargs" value=""/>
 	<property name="require.gettext" value="true" />
 
 	<target name="init">
-		<mkdir dir="${build}"/>
-        <mkdir dir="${build}/${resources}"/>
+	<mkdir dir="${build}"/>
+	<mkdir dir="${build}/${resources}"/>
         <mkdir dir="${build}/${javadoc}"/>
 		<mkdir dir="${dist}"/>
 	</target>
@@ -39,7 +42,7 @@
         </copy>
 	</target>
 
-	<target name="bundle" >
+	<target name="bundle" unless="no.bundle">
         <exec executable="sh" osfamily="unix" failifexecutionfails="true" failonerror="${require.gettext}" >
             <arg value="./bundle-messages.sh" />
         </exec>

apps/desktopgui/bundle-messages.sh

@@ -1,3 +1,4 @@
+#!/bin/sh
 #
 # Update messages_xx.po and messages_xx.class files,
 # from both java and jsp sources.

apps/desktopgui/locale/messages_el.po

@@ -0,0 +1,56 @@
+# I2P
+# Copyright (C) 2009 The I2P Project
+# This file is distributed under the same license as the desktopgui package.
+# To contribute translations, see http://www.i2p2.de/newdevelopers
+# 
+# Translators:
+#   <lixtetrax@grhack.net>, 2012.
+msgid ""
+msgstr ""
+"Project-Id-Version: I2P\n"
+"Report-Msgid-Bugs-To: https://trac.i2p2.de/\n"
+"POT-Creation-Date: 2011-03-03 18:29+0000\n"
+"PO-Revision-Date: 2012-07-02 11:28+0000\n"
+"Last-Translator: lixtetrax <lixtetrax@grhack.net>\n"
+"Language-Team: Greek (http://www.transifex.com/projects/p/I2P/language/el/)\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Language: el\n"
+"Plural-Forms: nplurals=2; plural=(n != 1)\n"
+
+#: src/net/i2p/desktopgui/ExternalTrayManager.java:23
+msgid "Start I2P"
+msgstr "Έναρξη Ι2Ρ"
+
+#: src/net/i2p/desktopgui/ExternalTrayManager.java:38
+msgid "I2P is starting!"
+msgstr "Το Ι2Ρ ξεκίνησε!"
+
+#: src/net/i2p/desktopgui/ExternalTrayManager.java:38
+msgid "Starting"
+msgstr "Έναρξη"
+
+#: src/net/i2p/desktopgui/InternalTrayManager.java:26
+msgid "Launch I2P Browser"
+msgstr "Έναρξη φυλλομετρητή Ι2Ρ"
+
+#: src/net/i2p/desktopgui/InternalTrayManager.java:50
+msgid "Configure desktopgui"
+msgstr "Παραμετροποίηση desktopgui"
+
+#: src/net/i2p/desktopgui/InternalTrayManager.java:67
+msgid "Restart I2P"
+msgstr "Επανεκκίνηση Ι2Ρ"
+
+#: src/net/i2p/desktopgui/InternalTrayManager.java:85
+msgid "Stop I2P"
+msgstr "Τερματισμός Ι2Ρ"
+
+#: src/net/i2p/desktopgui/gui/DesktopguiConfigurationFrame.java:44
+msgid "Tray icon configuration"
+msgstr "Παραμετροποίηση εικονιδίου"
+
+#: src/net/i2p/desktopgui/gui/DesktopguiConfigurationFrame.java:47
+msgid "Should tray icon be enabled?"
+msgstr "Ενεργοποίηση εικονιδίου;"

apps/desktopgui/locale/messages_hu.po

@@ -0,0 +1,55 @@
+# I2P
+# Copyright (C) 2009 The I2P Project
+# This file is distributed under the same license as the desktopgui package.
+# To contribute translations, see http://www.i2p2.de/newdevelopers
+# 
+# Translators:
+msgid ""
+msgstr ""
+"Project-Id-Version: I2P\n"
+"Report-Msgid-Bugs-To: https://trac.i2p2.de/\n"
+"POT-Creation-Date: 2011-03-03 18:29+0000\n"
+"PO-Revision-Date: 2012-06-01 16:28+0000\n"
+"Last-Translator: AdminLMH <lehetmashogy@i2pmail.org>\n"
+"Language-Team: Hungarian (http://www.transifex.net/projects/p/I2P/language/hu/)\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Language: hu\n"
+"Plural-Forms: nplurals=2; plural=(n != 1)\n"
+
+#: src/net/i2p/desktopgui/ExternalTrayManager.java:23
+msgid "Start I2P"
+msgstr "I2P indítása"
+
+#: src/net/i2p/desktopgui/ExternalTrayManager.java:38
+msgid "I2P is starting!"
+msgstr "I2P indul!"
+
+#: src/net/i2p/desktopgui/ExternalTrayManager.java:38
+msgid "Starting"
+msgstr "indítás"
+
+#: src/net/i2p/desktopgui/InternalTrayManager.java:26
+msgid "Launch I2P Browser"
+msgstr "I2P Böngésző Indítása"
+
+#: src/net/i2p/desktopgui/InternalTrayManager.java:50
+msgid "Configure desktopgui"
+msgstr "Asztali Grafikus Felület Beállítása"
+
+#: src/net/i2p/desktopgui/InternalTrayManager.java:67
+msgid "Restart I2P"
+msgstr "I2P Újraindítása"
+
+#: src/net/i2p/desktopgui/InternalTrayManager.java:85
+msgid "Stop I2P"
+msgstr "I2P Leállítása"
+
+#: src/net/i2p/desktopgui/gui/DesktopguiConfigurationFrame.java:44
+msgid "Tray icon configuration"
+msgstr "Tálcaikon beállítása"
+
+#: src/net/i2p/desktopgui/gui/DesktopguiConfigurationFrame.java:47
+msgid "Should tray icon be enabled?"
+msgstr "Tálcaikon engedélyezve legyen?"

apps/desktopgui/locale/messages_it.po

@@ -3,15 +3,17 @@
 # This file is distributed under the same license as the desktopgui package.
 # To contribute translations, see http://www.i2p2.de/newdevelopers
 # 
+# Translators:
+#   <bovas85@gmail.com>, 2012.
 #   <jokjok@hotmail.it>, 2011.
 msgid ""
 msgstr ""
 "Project-Id-Version: I2P\n"
 "Report-Msgid-Bugs-To: https://trac.i2p2.de/\n"
 "POT-Creation-Date: 2011-03-03 18:29+0000\n"
-"PO-Revision-Date: 2011-06-09 17:09+0000\n"
-"Last-Translator: mkkid <jokjok@hotmail.it>\n"
-"Language-Team: Italian (http://www.transifex.net/projects/p/I2P/team/it/)\n"
+"PO-Revision-Date: 2012-06-01 12:21+0000\n"
+"Last-Translator: Leelium <bovas85@gmail.com>\n"
+"Language-Team: Italian (http://www.transifex.net/projects/p/I2P/language/it/)\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
@@ -48,10 +50,8 @@ msgstr "Ferma I2P"
 
 #: src/net/i2p/desktopgui/gui/DesktopguiConfigurationFrame.java:44
 msgid "Tray icon configuration"
-msgstr "Configurazione dell'icona nell'Area di notifica"
+msgstr "Configurazione dell'icona nell'area di notifica"
 
 #: src/net/i2p/desktopgui/gui/DesktopguiConfigurationFrame.java:47
 msgid "Should tray icon be enabled?"
-msgstr "Vuoi che l'icona nell'Area di notifica venga abilitata?"
-
-
+msgstr "Vuoi che l'icona nelll'rea di notifica venga abilitata?"

apps/desktopgui/locale/messages_sv.po

@@ -3,7 +3,8 @@
 # This file is distributed under the same license as the desktopgui package.
 # To contribute translations, see http://www.i2p2.de/newdevelopers
 # 
-# 123hund123 <M8R-ra4r1r@mailinator.com>, 2011
+# Translators:
+# 123hund123 <M8R-ra4r1r@mailinator.com>, 2011.
 msgid ""
 msgstr ""
 "Project-Id-Version: I2P\n"
@@ -11,7 +12,7 @@ msgstr ""
 "POT-Creation-Date: 2011-03-03 18:29+0000\n"
 "PO-Revision-Date: 2011-03-22 15:49+0000\n"
 "Last-Translator: 123hund123 <M8R-ra4r1r@mailinator.com>\n"
-"Language-Team: Swedish (Sweden) (http://www.transifex.net/projects/p/I2P/team/sv_SE/)\n"
+"Language-Team: Swedish (Sweden) (http://www.transifex.net/projects/p/I2P/language/sv_SE/)\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
@@ -53,5 +54,3 @@ msgstr "Ikonpanelskonfiguration"
 #: src/net/i2p/desktopgui/gui/DesktopguiConfigurationFrame.java:47
 msgid "Should tray icon be enabled?"
 msgstr "Ska ikonpanelen vara aktiverad?"
-
-

apps/fortuna/build.xml

@@ -1,106 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project basedir="." default="all" name="fortuna">
-
-	<property name="cvs.base.dir" value="java/gnu-crypto" />
-	<property name="cvs.etc.dir" value="${cvs.base.dir}/etc" />
-	<property name="cvs.lib.dir" value="${cvs.base.dir}/lib" />
-	<property name="cvs.object.dir" value="${cvs.base.dir}/classes" />
-	<property name="cvs.base.crypto.object.dir" value="${cvs.object.dir}/gnu/crypto" />
-	<property name="cvs.cipher.object.dir" value="${cvs.base.crypto.object.dir}/cipher" />
-	<property name="cvs.hash.object.dir" value="${cvs.base.crypto.object.dir}/hash" />
-	<property name="cvs.prng.object.dir" value="${cvs.base.crypto.object.dir}/prng" />
-
-	<patternset id="fortuna.files">
-		<include name="${cvs.base.crypto.object.dir}/Registry.class"/>
-		<include name="${cvs.prng.object.dir}/Fortuna*.class"/>
-		<include name="${cvs.prng.object.dir}/BasePRNG.class"/>
-		<include name="${cvs.prng.object.dir}/RandomEventListener.class"/>
-		<include name="${cvs.prng.object.dir}/IRandom.class"/>
-		<include name="${cvs.cipher.object.dir}/CipherFactory.class"/>
-		<include name="${cvs.cipher.object.dir}/IBlockCipher.class"/>
-		<include name="${cvs.hash.object.dir}/HashFactory.class"/>
-		<include name="${cvs.hash.object.dir}/IMessageDigest.class"/>
-	</patternset>
-
-	<target name="all" depends="build,jar" 
-		description="Create and test the custom Fortuna library" />
-
-	<target name="build" depends="-init,checkout"
-		description="Build the source and tests">
-		<ant dir="${cvs.base.dir}" target="jar" />
-	</target>
-
-	<target name="builddep" />
-
-	<target name="checkout" depends="-init" unless="cvs.source.available"
-		description="Check out GNU Crypto sources from CVS HEAD">
-		<cvs cvsRoot=":ext:anoncvs@savannah.gnu.org:/cvsroot/gnu-crypto"
-			cvsRsh="ssh"
-			dest="java"
-			package="gnu-crypto" />
-	</target>
-
-	<target name="clean"
-		description="Remove generated tests and object files">
-		<ant dir="${cvs.base.dir}" target="clean" />
-	</target>
-
-	<target name="cleandep" />
-
-	<target name="compile" />
-
-	<target name="distclean" depends="clean"
-		description="Remove all generated files">
-		<delete dir="build" />
-		<delete dir="jartemp" />
-		<!--
-			Annoyingly the GNU Crypto distclean task called here doesn't clean
-			*all* derived files from java/gnu-crypto/lib like it should.....
-		-->
-		<ant dir="${cvs.base.dir}" target="distclean" />
-		<!--
-			.....and so we mop up the rest ourselves.
-		-->
-		<delete dir="${cvs.lib.dir}" />
-	</target>
-
-	<target name="-init">
-		<available property="cvs.source.available" file="${cvs.base.dir}" />
-	</target>
-
-	<target name="jar" depends="build"
-		description="Create the custom Fortuna jar library">
-		<delete dir="build" />
-		<delete dir="jartemp" />
-		<mkdir dir="build" />
-		<mkdir dir="jartemp/${cvs.object.dir}" />
-		<copy todir="jartemp">
-			<fileset dir=".">
-				<patternset refid="fortuna.files" />
-			</fileset>
-		</copy>
-		<jar basedir="jartemp/${cvs.object.dir}" jarfile="build/fortuna.jar">
-			<manifest>
-				<section name="fortuna">
-					<attribute name="Implementation-Title" value="I2P Custom GNU Crypto Fortuna Library" />
-					<attribute name="Implementation-Version" value="CVS HEAD" /> 
-					<attribute name="Implementation-Vendor" value="Free Software Foundation" />
-					<attribute name="Implementation-Vendor-Id" value="FSF" />
-					<attribute name="Implementation-URL" value="http://www.gnu.org/software/gnu-crypto" />
-				</section>
-			</manifest>
-		</jar>
-		<delete dir="jartemp" />
-	</target>
-
-	<target name="test" depends="jar"
-		description="Perform crypto tests on custom Fortuna jar library" />
-		<!--
-			Add this when Fortuna tests are added to GNU Crypto, else write some
-		-->
-
-	<target name="update" depends="checkout"
-		description="Update GNU Crypto sources to latest CVS HEAD">
-		<cvs command="update -d" cvsRsh="ssh" dest="java/gnu-crypto" />
-	</target>
-</project>

apps/i2psnark/.classpath

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry kind="src" path="java/src"/>
+	<classpathentry combineaccessrules="false" kind="src" path="/i2p_sdk"/>
+	<classpathentry combineaccessrules="false" kind="src" path="/ministreaming"/>
+	<classpathentry kind="lib" path="/jetty/jettylib/javax.servlet.jar"/>
+	<classpathentry kind="lib" path="/jetty/jettylib/jetty-util.jar"/>
+	<classpathentry kind="lib" path="/jetty/jettylib/org.mortbay.jetty.jar"/>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+	<classpathentry kind="output" path="java/build/obj"/>
+</classpath>

apps/i2psnark/.project

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>i2psnark</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.jdt.core.javabuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.jdt.core.javanature</nature>
+	</natures>
+</projectDescription>

apps/i2psnark/java/build.xml

@@ -19,10 +19,14 @@
                 <pathelement location="../../ministreaming/java/build/obj" />
                 <pathelement location="../../jetty/jettylib/org.mortbay.jetty.jar" />
                 <pathelement location="../../jetty/jettylib/javax.servlet.jar" />
+                <pathelement location="../../jetty/jettylib/jetty-util.jar" />
             </classpath>
         </depend>
     </target>
 
+    <condition property="no.bundle">
+        <isfalse value="${require.gettext}" />
+    </condition>
     <property name="javac.compilerargs" value="" />
     <property name="require.gettext" value="true" />
 
@@ -34,7 +38,7 @@
             debug="true" deprecation="on" source="1.5" target="1.5" 
             destdir="./build/obj" 
             includeAntRuntime="false"
-            classpath="../../../core/java/build/i2p.jar:../../jetty/jettylib/org.mortbay.jetty.jar:../../jetty/jettylib/javax.servlet.jar:../../ministreaming/java/build/mstreaming.jar" >
+            classpath="../../../core/java/build/i2p.jar:../../jetty/jettylib/org.mortbay.jetty.jar:../../jetty/jettylib/javax.servlet.jar:../../jetty/jettylib/jetty-util.jar:../../ministreaming/java/build/mstreaming.jar" >
             <compilerarg line="${javac.compilerargs}" />
         </javac>
     </target>
@@ -56,11 +60,12 @@
     <target name="jar" depends="builddep, compile, jarUpToDate, listChangedFiles" unless="jar.uptodate" >
         <!-- set if unset -->
         <property name="workspace.changes.tr" value="" />
-        <jar destfile="./build/i2psnark.jar" basedir="./build/obj" includes="**/*.class" excludes="**/I2PSnarkServlet*.class **/messages_*.class">
+        <jar destfile="./build/i2psnark.jar" basedir="./build/obj" includes="**/*.class" excludes="**/I2PSnarkServlet*.class **/FetchAndAdd*.class **/messages_*.class">
             <manifest>
                 <attribute name="Main-Class" value="org.klomp.snark.Snark" />
                 <attribute name="Class-Path" value="i2p.jar mstreaming.jar streaming.jar" />
                 <attribute name="Implementation-Version" value="${full.version}" />
+                <attribute name="Built-By" value="${build.built-by}" />
                 <attribute name="Build-Date" value="${build.timestamp}" />
                 <attribute name="Base-Revision" value="${workspace.version}" />
                 <attribute name="Workspace-Changes" value="${workspace.changes.tr}" />
@@ -70,7 +75,7 @@
     
     <target name="jarUpToDate">
         <uptodate property="jar.uptodate" targetfile="build/i2psnark.jar" >
-            <srcfiles dir= "build/obj" includes="**/*.class" excludes="**/I2PSnarkServlet*.class **/messages_*.class" />
+            <srcfiles dir= "build/obj" includes="**/*.class" excludes="**/I2PSnarkServlet*.class **/FetchAndAdd*.class **/messages_*.class" />
         </uptodate>
         <condition property="shouldListChanges" >
             <and>
@@ -78,7 +83,7 @@
                     <isset property="jar.uptodate" />
                 </not>
                 <not>
-                    <isset property="wjar.uptodate" />
+                    <isset property="war.uptodate" />
                 </not>
                 <isset property="mtn.available" />
             </and>
@@ -95,11 +100,16 @@
     <target name="war" depends="jar, bundle, warUpToDate, listChangedFiles" unless="war.uptodate" > 
         <!-- set if unset -->
         <property name="workspace.changes.tr" value="" />
-        <war destfile="../i2psnark.war" webxml="../web.xml" basedir="../" includes="_icons/*" >
+        <copy todir="build/icons/.icons" >
+            <fileset dir="../icons/" />
+        </copy>
+        <war destfile="../i2psnark.war" webxml="../web.xml" >
           <!-- include only the web stuff, as of 0.7.12 the router will add i2psnark.jar to the classpath for the war -->
           <classes dir="./build/obj" includes="**/web/*.class" />
+            <fileset dir="build/icons/" />
             <manifest>
                 <attribute name="Implementation-Version" value="${full.version}" />
+                <attribute name="Built-By" value="${build.built-by}" />
                 <attribute name="Build-Date" value="${build.timestamp}" />
                 <attribute name="Base-Revision" value="${workspace.version}" />
                 <attribute name="Workspace-Changes" value="${workspace.changes.tr}" />
@@ -109,11 +119,11 @@
     
     <target name="warUpToDate">
         <uptodate property="war.uptodate" targetfile="../i2psnark.war" >
-            <srcfiles dir= "." includes="build/obj/org/klomp/snark/web/*.class ../_icons/* ../web.xml" />
+            <srcfiles dir= "." includes="build/obj/org/klomp/snark/web/*.class ../icons/* ../web.xml" />
         </uptodate>
     </target>
     
-    <target name="bundle" depends="compile">
+    <target name="bundle" depends="compile" unless="no.bundle">
         <!-- Update the messages_*.po files.
              We need to supply the bat file for windows, and then change the fail property to true -->
         <exec executable="sh" osfamily="unix" failifexecutionfails="true" failonerror="${require.gettext}" >

apps/i2psnark/java/bundle-messages.sh

@@ -1,3 +1,4 @@
+#!/bin/sh
 #
 # Update messages_xx.po and messages_xx.class files,
 # from both java and jsp sources.

apps/i2psnark/java/src/net/i2p/kademlia/KBucket.java

@@ -0,0 +1,76 @@
+package net.i2p.kademlia;
+/*
+ * free (adj.): unencumbered; not under the control of others
+ * Written by jrandom in 2003 and released into the public domain 
+ * with no warranty of any kind, either expressed or implied.  
+ * It probably won't make your computer catch on fire, or eat 
+ * your children, but it might.  Use at your own risk.
+ *
+ */
+
+import java.util.Set;
+
+import net.i2p.data.SimpleDataStructure;
+
+/**
+ * Group, without inherent ordering, a set of keys a certain distance away from
+ * a local key, using XOR as the distance metric
+ *
+ * Refactored from net.i2p.router.networkdb.kademlia
+ * @since 0.9.2
+ */
+public interface KBucket<T extends SimpleDataStructure> {
+
+    /** 
+     * Lowest order high bit for difference keys.
+     * The lower-bounds distance of this bucket is 2**begin.
+     * If begin == 0, this is the closest bucket.
+     */
+    public int getRangeBegin();
+
+    /**
+     * Highest high bit for the difference keys.
+     * The upper-bounds distance of this bucket is (2**(end+1)) - 1.
+     * If begin == end, the bucket cannot be split further.
+     * If end == (numbits - 1), this is the furthest bucket.
+     */
+    public int getRangeEnd();
+
+    /**
+     * Number of keys already contained in this kbucket
+     */
+    public int getKeyCount();
+
+    /**
+     * Add the peer to the bucket
+     *
+     * @return true if added
+     */
+    public boolean add(T key);
+
+    /**
+     * Remove the key from the bucket
+     * @return true if the key existed in the bucket before removing it, else false
+     */
+    public boolean remove(T key);
+    
+    /**
+     *  Update the last-changed timestamp to now.
+     */
+    public void setLastChanged();
+
+    /**
+     *  The last-changed timestamp
+     */
+    public long getLastChanged();
+
+    /**
+     * Retrieve all routing table entries stored in the bucket
+     * @return set of Hash structures
+     */
+    public Set<T> getEntries();
+
+    public void getEntries(SelectionCollector<T> collector);
+
+    public void clear();
+}

apps/i2psnark/java/src/net/i2p/kademlia/KBucketImpl.java

@@ -0,0 +1,150 @@
+package net.i2p.kademlia;
+/*
+ * free (adj.): unencumbered; not under the control of others
+ * Written by jrandom in 2003 and released into the public domain
+ * with no warranty of any kind, either expressed or implied.
+ * It probably won't make your computer catch on fire, or eat
+ * your children, but it might.  Use at your own risk.
+ *
+ */
+
+import java.util.Collections;
+import java.util.Set;
+
+import net.i2p.I2PAppContext;
+import net.i2p.data.SimpleDataStructure;
+import net.i2p.util.ConcurrentHashSet;
+
+/**
+ *  A concurrent implementation using ConcurrentHashSet.
+ *  The max size (K) may be temporarily exceeded due to concurrency,
+ *  a pending split, or the behavior of the supplied trimmer,
+ *  as explained below.
+ *  The creator is responsible for splits.
+ *
+ *  This class has no knowledge of the DHT base used for XORing,
+ *  and thus there are no validity checks in add/remove.
+ *
+ *  The begin and end values are immutable.
+ *  All entries in this bucket will have at least one bit different
+ *  from us in the range [begin, end] inclusive.
+ *  Splits must be implemented by creating two new buckets
+ *  and discarding this one.
+ *
+ *  The keys are kept in a Set and are NOT sorted by last-seen.
+ *  Per-key last-seen-time, failures, etc. must be tracked elsewhere.
+ *
+ *  If this bucket is full (i.e. begin == end && size == max)
+ *  then add() will call KBucketTrimmer.trim() do
+ *  (possibly) remove older entries, and indicate whether
+ *  to add the new entry. If the trimmer returns true without
+ *  removing entries, this KBucket will exceed the max size.
+ *
+ *  Refactored from net.i2p.router.networkdb.kademlia
+ *  @since 0.9.2
+ */
+class KBucketImpl<T extends SimpleDataStructure> implements KBucket<T> {
+    /**
+     *  set of Hash objects for the peers in the kbucket
+     */
+    private final Set<T> _entries;
+    /** include if any bits equal or higher to this bit (in big endian order) */
+    private final int _begin;
+    /** include if no bits higher than this bit (inclusive) are set */
+    private final int _end;
+    private final int _max;
+    private final KBucketTrimmer _trimmer;
+    /** when did we last shake things up */
+    private long _lastChanged;
+    private final I2PAppContext _context;
+    
+    /**
+     *  All entries in this bucket will have at least one bit different
+     *  from us in the range [begin, end] inclusive.
+     */
+    public KBucketImpl(I2PAppContext context, int begin, int end, int max, KBucketTrimmer trimmer) {
+        if (begin > end)
+            throw new IllegalArgumentException(begin + " > " + end);
+        _context = context;
+        _entries = new ConcurrentHashSet(max + 4);
+        _begin = begin;
+        _end = end;
+        _max = max;
+        _trimmer = trimmer;
+    }
+    
+    public int getRangeBegin() { return _begin; }
+
+    public int getRangeEnd() { return _end; }
+
+    public int getKeyCount() {
+        return _entries.size();
+    }
+    
+    /**
+     *  @return an unmodifiable view; not a copy
+     */
+    public Set<T> getEntries() {
+        return Collections.unmodifiableSet(_entries);
+    }
+
+    public void getEntries(SelectionCollector<T> collector) {
+        for (T h : _entries) {
+             collector.add(h);
+        }
+    }
+    
+    public void clear() {
+        _entries.clear();
+    }
+    
+    /**
+     *  Sets last-changed if rv is true OR if the peer is already present.
+     *  Calls the trimmer if begin == end and we are full.
+     *  If begin != end then add it and caller must do bucket splitting.
+     *  @return true if added
+     */
+    public boolean add(T peer) {
+        if (_begin != _end || _entries.size() < _max ||
+            _entries.contains(peer) || _trimmer.trim(this, peer)) {
+            // do this even if already contains, to call setLastChanged()
+            boolean rv = _entries.add(peer);
+            setLastChanged();
+            return rv;
+        }
+        return false;
+    }
+    
+    /**
+     *  @return if removed. Does NOT set lastChanged.
+     */
+    public boolean remove(T peer) {
+        boolean rv = _entries.remove(peer);
+        //if (rv)
+        //    setLastChanged();
+        return rv;
+    }
+    
+    /**
+     *  Update the last-changed timestamp to now.
+     */
+    public void setLastChanged() {
+        _lastChanged = _context.clock().now();
+    }
+
+    /**
+     *  The last-changed timestamp, which actually indicates last-added or last-seen.
+     */
+    public long getLastChanged() {
+        return _lastChanged;
+    }
+    
+    @Override
+    public String toString() {
+        StringBuilder buf = new StringBuilder(1024);
+        buf.append(_entries.size());
+        buf.append(" entries in (").append(_begin).append(',').append(_end);
+        buf.append(") : ").append(_entries.toString());
+        return buf.toString();
+    }
+}

apps/i2psnark/java/src/net/i2p/kademlia/KBucketSet.java

@@ -0,0 +1,780 @@
+package net.i2p.kademlia;
+/*
+ * free (adj.): unencumbered; not under the control of others
+ * Written by jrandom in 2003 and released into the public domain
+ * with no warranty of any kind, either expressed or implied.
+ * It probably won't make your computer catch on fire, or eat
+ * your children, but it might.  Use at your own risk.
+ *
+ */
+
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import net.i2p.I2PAppContext;
+import net.i2p.data.DataHelper;
+import net.i2p.data.SimpleDataStructure;
+import net.i2p.util.LHMCache;
+import net.i2p.util.Log;
+
+/**
+ * In-memory storage of buckets sorted by the XOR metric from the base (us)
+ * passed in via the constructor.
+ * This starts with one bucket covering the whole key space, and
+ * may eventually be split to a max of the number of bits in the data type
+ * (160 for SHA1Hash or 256 for Hash),
+ * times 2**(B-1) for Kademlia value B.
+ *
+ * Refactored from net.i2p.router.networkdb.kademlia
+ * @since 0.9.2
+ */
+public class KBucketSet<T extends SimpleDataStructure> {
+    private final Log _log;
+    private final I2PAppContext _context;
+    private final T _us;
+
+    /**
+     * The bucket list is locked by _bucketsLock, however the individual
+     * buckets are not locked. Users may see buckets that have more than
+     * the maximum k entries, or may have adds and removes silently fail
+     * when they appear to succeed.
+     *
+     * Closest values are in bucket 0, furthest are in the last bucket.
+     */
+    private final List<KBucket> _buckets;
+    private final Range<T> _rangeCalc;
+    private final KBucketTrimmer _trimmer;
+    
+    /**
+     *  Locked for reading only when traversing all the buckets.
+     *  Locked for writing only when splitting a bucket.
+     *  Adds/removes/gets from individual buckets are not locked.
+     */
+    private final ReentrantReadWriteLock _bucketsLock = new ReentrantReadWriteLock(false);
+
+    private final int KEYSIZE_BITS;
+    private final int NUM_BUCKETS;
+    private final int BUCKET_SIZE;
+    private final int B_VALUE;
+    private final int B_FACTOR;
+    
+    /**
+     * Use the default trim strategy, which removes a random entry.
+     * @param us the local identity (typically a SHA1Hash or Hash)
+     *           The class must have a zero-argument constructor.
+     * @param max the Kademlia value "k", the max per bucket, k >= 4
+     * @param b the Kademlia value "b", split buckets an extra 2**(b-1) times,
+     *           b > 0, use 1 for bittorrent, Kademlia paper recommends 5
+     */
+    public KBucketSet(I2PAppContext context, T us, int max, int b) {
+        this(context, us, max, b, new RandomTrimmer(context, max));
+    }
+
+    /**
+     * Use the supplied trim strategy.
+     */
+    public KBucketSet(I2PAppContext context, T us, int max, int b, KBucketTrimmer trimmer) {
+        _us = us;
+        _context = context;
+        _log = context.logManager().getLog(KBucketSet.class);
+        _trimmer = trimmer;
+        if (max <= 4 || b <= 0 || b > 8)
+            throw new IllegalArgumentException();
+        KEYSIZE_BITS = us.length() * 8;
+        B_VALUE = b;
+        B_FACTOR = 1 << (b - 1);
+        NUM_BUCKETS = KEYSIZE_BITS * B_FACTOR;
+        BUCKET_SIZE = max;
+        _buckets = createBuckets();
+        _rangeCalc = new Range(us, B_VALUE);
+        // this verifies the zero-argument constructor
+        makeKey(new byte[us.length()]);
+    }
+    
+    private void getReadLock() {
+        _bucketsLock.readLock().lock();
+    }
+
+    /**
+     *  Get the lock if we can. Non-blocking.
+     *  @return true if the lock was acquired
+     */
+    private boolean tryReadLock() {
+        return _bucketsLock.readLock().tryLock();
+    }
+
+    private void releaseReadLock() {
+        _bucketsLock.readLock().unlock();
+    }
+
+    /** @return true if the lock was acquired */
+    private boolean getWriteLock() {
+        try {
+            boolean rv = _bucketsLock.writeLock().tryLock(3000, TimeUnit.MILLISECONDS);
+            if ((!rv) && _log.shouldLog(Log.WARN))
+                _log.warn("no lock, size is: " + _bucketsLock.getQueueLength(), new Exception("rats"));
+            return rv;
+        } catch (InterruptedException ie) {}
+        return false;
+    }
+
+    private void releaseWriteLock() {
+        _bucketsLock.writeLock().unlock();
+    }
+
+    /**
+     * @return true if the peer is new to the bucket it goes in, or false if it was
+     *  already in it. Always returns false on an attempt to add ourselves.
+     *
+     */
+    public boolean add(T peer) {
+        KBucket bucket;
+        getReadLock();
+        try {
+            bucket = getBucket(peer);
+        } finally { releaseReadLock(); }
+        if (bucket != null) {
+            if (bucket.add(peer)) {
+                if (_log.shouldLog(Log.DEBUG))
+                    _log.debug("Peer " + peer + " added to bucket " + bucket);
+                if (shouldSplit(bucket)) {
+                    if (_log.shouldLog(Log.DEBUG))
+                        _log.debug("Splitting bucket " + bucket);
+                    split(bucket.getRangeBegin());
+                    //testAudit(this, _log);
+                }
+                return true;
+            } else {
+                if (_log.shouldLog(Log.DEBUG))
+                    _log.debug("Peer " + peer + " NOT added to bucket " + bucket);
+                return false;
+            }
+        } else {
+            if (_log.shouldLog(Log.WARN))
+                _log.warn("Failed to add, probably us: " + peer);
+            return false;
+        }
+    }
+
+    /**
+     *  No lock required.
+     *  FIXME will split the closest buckets too far if B > 1 and K < 2**B
+     *  Won't ever really happen and if it does it still works.
+     */
+    private boolean shouldSplit(KBucket b) {
+        return
+               b.getRangeBegin() != b.getRangeEnd() &&
+               b.getKeyCount() > BUCKET_SIZE;
+    }
+
+    /**
+     *  Grabs the write lock.
+     *  Caller must NOT have the read lock.
+     *  The bucket should be splittable (range start != range end).
+     *  @param r the range start of the bucket to be split
+     */
+    private void split(int r) {
+        if (!getWriteLock())
+            return;
+        try {
+            locked_split(r);
+        } finally { releaseWriteLock(); }
+    }
+
+    /**
+     *  Creates two or more new buckets. The old bucket is replaced and discarded.
+     *
+     *  Caller must hold write lock
+     *  The bucket should be splittable (range start != range end).
+     *  @param r the range start of the bucket to be split
+     */
+    private void locked_split(int r) {
+        int b = pickBucket(r);
+        while (shouldSplit(_buckets.get(b))) {
+            KBucket<T> b0 = _buckets.get(b);
+            // Each bucket gets half the keyspace.
+            // When B_VALUE = 1, or the bucket is larger than B_FACTOR, then
+            // e.g. 0-159 => 0-158, 159-159
+            // When B_VALUE > 1, and the bucket is smaller than B_FACTOR, then
+            // e.g. 1020-1023 => 1020-1021, 1022-1023
+            int s1, e1, s2, e2;
+            s1 = b0.getRangeBegin();
+            e2 = b0.getRangeEnd();
+            if (B_VALUE == 1 ||
+                ((s1 & (B_FACTOR - 1)) == 0 &&
+                 ((e2 + 1) & (B_FACTOR - 1)) == 0 &&
+                 e2 > s1 + B_FACTOR)) {
+                // The bucket is a "whole" kbucket with a range > B_FACTOR,
+                // so it should be split into two "whole" kbuckets each with
+                // a range >= B_FACTOR.
+                // Log split
+                s2 = e2 + 1 - B_FACTOR;
+            } else {
+                // The bucket is the smallest "whole" kbucket with a range == B_FACTOR,
+                // or B_VALUE > 1 and the bucket has already been split.
+                // Start or continue splitting down to a depth B_VALUE.
+                // Linear split
+                s2 = s1 + ((1 + e2 - s1) / 2);
+            }	
+            e1 = s2 - 1;
+            if (_log.shouldLog(Log.INFO))
+                _log.info("Splitting (" + s1 + ',' + e2 + ") -> (" + s1 + ',' + e1 + ") (" + s2 + ',' + e2 + ')');
+            KBucket<T> b1 = createBucket(s1, e1);
+            KBucket<T> b2 = createBucket(s2, e2);
+            for (T key : b0.getEntries()) {
+                if (getRange(key) < s2)
+                    b1.add(key);
+                else
+                    b2.add(key);
+            }
+            _buckets.set(b, b1);
+            _buckets.add(b + 1, b2);
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("Split bucket at idx " + b +
+                           ":\n" + b0 +
+                           "\ninto: " + b1 +
+                           "\nand: " + b2);
+            //if (_log.shouldLog(Log.DEBUG))
+            //    _log.debug("State is now: " + toString());
+
+            if (b2.getKeyCount() > BUCKET_SIZE) {
+                // should be rare... too hard to call _trimmer from here
+                // (and definitely not from inside the write lock)
+                if (_log.shouldLog(Log.INFO))
+                    _log.info("All went into 2nd bucket after split");
+            }
+            // loop if all the entries went in the first bucket
+        }
+    }
+
+    /**
+     *  The current number of entries.
+     */
+    public int size() {
+        int rv = 0;
+        getReadLock();
+        try {
+            for (KBucket b : _buckets) {
+                rv += b.getKeyCount();
+            }
+        } finally { releaseReadLock(); }
+        return rv;
+    }
+    
+    public boolean remove(T entry) {
+        KBucket kbucket;
+        getReadLock();
+        try {
+            kbucket = getBucket(entry);
+        } finally { releaseReadLock(); }
+        boolean removed = kbucket.remove(entry);
+        return removed;
+    }
+    
+    /** @since 0.8.8 */
+    public void clear() {
+        getReadLock();
+        try {
+            for (KBucket b : _buckets) {
+                b.clear();
+            }
+        } finally { releaseReadLock(); }
+        _rangeCalc.clear();
+    }
+    
+    /**
+     *  @return a copy in a new set
+     */
+    public Set<T> getAll() {
+        Set<T> all = new HashSet(256);
+        getReadLock();
+        try {
+            for (KBucket b : _buckets) {
+                all.addAll(b.getEntries());
+            }
+        } finally { releaseReadLock(); }
+        return all;
+    }
+
+    /**
+     *  @return a copy in a new set
+     */
+    public Set<T> getAll(Set<T> toIgnore) {
+        Set<T> all = getAll();
+        all.removeAll(toIgnore);
+        return all;
+    }
+    
+    public void getAll(SelectionCollector<T> collector) {
+        getReadLock();
+        try {
+            for (KBucket b : _buckets) {
+                b.getEntries(collector);
+            }
+        } finally { releaseReadLock(); }
+    }
+    
+    /**
+     *  The keys closest to us.
+     *  Returned list will never contain us.
+     *  @return non-null, closest first
+     */
+    public List<T> getClosest(int max) {
+        return getClosest(max, Collections.EMPTY_SET);
+    }
+    
+    /**
+     *  The keys closest to us.
+     *  Returned list will never contain us.
+     *  @return non-null, closest first
+     */
+    public List<T> getClosest(int max, Collection<T> toIgnore) {
+        List<T> rv = new ArrayList(max);
+        int count = 0;
+        getReadLock();
+        try {
+            // start at first (closest) bucket
+            for (int i = 0; i < _buckets.size() && count < max; i++) {
+                Set<T> entries = _buckets.get(i).getEntries();
+                // add the whole bucket except for ignores,
+                // extras will be trimmed after sorting
+                for (T e : entries) {
+                    if (!toIgnore.contains(e)) {
+                        rv.add(e);
+                        count++;
+                    }
+                }
+            }
+        } finally { releaseReadLock(); }
+        Comparator comp = new XORComparator(_us);
+        Collections.sort(rv, comp);
+        int sz = rv.size();
+        for (int i = sz - 1; i >= max; i--) {
+            rv.remove(i);
+        }
+        return rv;
+    }
+    
+    /**
+     *  The keys closest to the key.
+     *  Returned list will never contain us.
+     *  @return non-null, closest first
+     */
+    public List<T> getClosest(T key, int max) {
+        return getClosest(key, max, Collections.EMPTY_SET);
+    }
+    
+    /**
+     *  The keys closest to the key.
+     *  Returned list will never contain us.
+     *  @return non-null, closest first
+     */
+    public List<T> getClosest(T key, int max, Collection<T> toIgnore) {
+        if (key.equals(_us))
+            return getClosest(max, toIgnore);
+        List<T> rv = new ArrayList(max);
+        int count = 0;
+        getReadLock();
+        try {
+            int start = pickBucket(key);
+            // start at closest bucket, then to the smaller (closer to us) buckets
+            for (int i = start; i >= 0 && count < max; i--) {
+                Set<T> entries = _buckets.get(i).getEntries();
+                for (T e : entries) {
+                    if (!toIgnore.contains(e)) {
+                        rv.add(e);
+                        count++;
+                    }
+                }
+            }
+            // then the farther from us buckets if necessary
+            for (int i = start + 1; i < _buckets.size() && count < max; i++) {
+                Set<T> entries = _buckets.get(i).getEntries();
+                for (T e : entries) {
+                    if (!toIgnore.contains(e)) {
+                        rv.add(e);
+                        count++;
+                    }
+                }
+            }
+        } finally { releaseReadLock(); }
+        Comparator comp = new XORComparator(key);
+        Collections.sort(rv, comp);
+        int sz = rv.size();
+        for (int i = sz - 1; i >= max; i--) {
+            rv.remove(i);
+        }
+        return rv;
+    }
+
+    /**
+     *  The bucket number (NOT the range number) that the xor of the key goes in
+     *  Caller must hold read lock
+     *  @return 0 to max-1 or -1 for us
+     */
+    private int pickBucket(T key) {
+        int range = getRange(key);
+        if (range < 0)
+            return -1;
+        int rv = pickBucket(range);
+        if (rv >= 0) {
+             return rv;
+        }
+        _log.error("Key does not fit in any bucket?! WTF!\nKey  : [" 
+                   + DataHelper.toHexString(key.getData()) + "]" 
+                   + "\nUs   : " + _us
+                   + "\nDelta: ["
+                   + DataHelper.toHexString(DataHelper.xor(_us.getData(), key.getData()))
+                   + "]", new Exception("WTF"));
+        _log.error(toString());
+        throw new IllegalStateException("pickBucket returned " + rv);
+        //return -1;
+    }
+    
+    /**
+     *  Returned list is a copy of the bucket list, closest first,
+     *  with the actual buckets (not a copy).
+     *
+     *  Primarily for testing. You shouldn't ever need to get all the buckets.
+     *  Use getClosest() or getAll() instead to get the keys.
+     *
+     *  @return non-null
+     */
+    List<KBucket<T>> getBuckets() {
+        getReadLock();
+        try {
+            return new ArrayList(_buckets);
+        } finally { releaseReadLock(); }
+    }
+
+    /**
+     *  The bucket that the xor of the key goes in
+     *  Caller must hold read lock
+     *  @return null if key is us
+     */
+    private KBucket getBucket(T key) {
+       int bucket = pickBucket(key);
+       if (bucket < 0)
+           return null;
+       return _buckets.get(bucket);
+    }
+    
+    /**
+     *  The bucket number that contains this range number
+     *  Caller must hold read lock or write lock
+     *  @return 0 to max-1 or -1 for us
+     */
+    private int pickBucket(int range) {
+        // If B is small, a linear search from back to front
+        // is most efficient since most of the keys are at the end...
+        // If B is larger, there's a lot of sub-buckets
+        // of equal size to be checked so a binary search is better
+        if (B_VALUE <= 3) {
+            for (int i = _buckets.size() - 1; i >= 0; i--) {
+                KBucket b = _buckets.get(i);
+                if (range >= b.getRangeBegin() && range <= b.getRangeEnd())
+                    return i;
+            }
+            return -1;
+        } else {
+            KBucket dummy = new DummyBucket(range);
+            return Collections.binarySearch(_buckets, dummy, new BucketComparator());
+        }
+    }
+
+    private List<KBucket> createBuckets() {
+        // just an initial size
+        List<KBucket> buckets = new ArrayList(4 * B_FACTOR);
+        buckets.add(createBucket(0, NUM_BUCKETS -1));
+        return buckets;
+    }
+    
+    private KBucket createBucket(int start, int end) {
+        if (end - start >= B_FACTOR &&
+            (((end + 1) & B_FACTOR - 1) != 0 ||
+             (start & B_FACTOR - 1) != 0))
+            throw new IllegalArgumentException("Sub-bkt crosses K-bkt boundary: " + start + '-' + end);
+        KBucket bucket = new KBucketImpl(_context, start, end, BUCKET_SIZE, _trimmer);
+        return bucket;
+    }
+    
+    /**
+     *  The number of bits minus 1 (range number) for the xor of the key.
+     *  Package private for testing only. Others shouldn't need this.
+     *  @return 0 to max-1 or -1 for us
+     */
+    int getRange(T key) {
+        return _rangeCalc.getRange(key);
+    }
+    
+    /**
+     *  For every bucket that hasn't been updated in this long,
+     *  or isn't close to full,
+     *  generate a random key that would be a member of that bucket.
+     *  The returned keys may be searched for to "refresh" the buckets.
+     *  @return non-null, closest first
+     */
+    public List<T> getExploreKeys(long age) {
+        List<T> rv = new ArrayList(_buckets.size());
+        long old = _context.clock().now() - age;
+        getReadLock();
+        try {
+            for (KBucket b : _buckets) {
+                int curSize = b.getKeyCount();
+                // Always explore the closest bucket
+                if ((b.getRangeBegin() == 0) ||
+                    (b.getLastChanged() < old || curSize < BUCKET_SIZE * 3 / 4))
+                    rv.add(generateRandomKey(b));
+            }
+        } finally { releaseReadLock(); }
+        return rv;
+    }
+    
+    /**
+     *  Generate a random key to go within this bucket
+     *  Package private for testing only. Others shouldn't need this.
+     */
+    T generateRandomKey(KBucket bucket) {
+        int begin = bucket.getRangeBegin();
+        int end = bucket.getRangeEnd();
+        // number of fixed bits, out of B_VALUE - 1 bits
+        int fixed = 0;
+        int bsz = 1 + end - begin;
+        // compute fixed = B_VALUE - log2(bsz)
+        // e.g for B=4, B_FACTOR=8, sz 4-> fixed 1, sz 2->fixed 2, sz 1 -> fixed 3
+        while (bsz < B_FACTOR) {
+            fixed++;
+            bsz <<= 1;
+        }
+        int fixedBits = 0;
+        if (fixed > 0) {
+            // 0x01, 03, 07, 0f, ...
+            int mask = (1 << fixed) - 1;
+            // fixed bits masked from begin
+            fixedBits = (begin >> (B_VALUE - (fixed + 1))) & mask;
+        }
+        int obegin = begin;
+        int oend = end;
+        begin >>= (B_VALUE - 1);
+        end >>= (B_VALUE - 1);
+        // we need randomness for [0, begin) bits
+        BigInteger variance;
+        // 00000000rrrr
+        if (begin > 0)
+            variance = new BigInteger(begin - fixed, _context.random());
+        else
+            variance = BigInteger.ZERO;
+        // we need nonzero randomness for [begin, end] bits
+        int numNonZero = 1 + end - begin;
+        if (numNonZero == 1) {
+            // 00001000rrrr
+            variance = variance.setBit(begin);
+            // fixed bits as the 'main' bucket is split
+            // 00001fffrrrr
+            if (fixed > 0)
+                variance = variance.or(BigInteger.valueOf(fixedBits).shiftLeft(begin - fixed));
+        } else {
+            // dont span main bucket boundaries with depth > 1
+            if (fixed > 0)
+                throw new IllegalStateException("WTF " + bucket);
+            BigInteger nonz;
+            if (numNonZero <= 62) {
+                // add one to ensure nonzero
+                long nz = 1 + _context.random().nextLong((1l << numNonZero) - 1);
+                nonz = BigInteger.valueOf(nz);
+            } else {
+                // loop to ensure nonzero
+                do {
+                    nonz = new BigInteger(numNonZero, _context.random());
+                } while (nonz.equals(BigInteger.ZERO));
+            }
+            // shift left and or-in the nonzero randomness
+            if (begin > 0)
+                nonz = nonz.shiftLeft(begin);
+            // 0000nnnnrrrr
+            variance = variance.or(nonz);
+        }
+
+        if (_log.shouldLog(Log.DEBUG))
+            _log.debug("SB(" + obegin + ',' + oend + ") KB(" + begin + ',' + end + ") fixed=" + fixed + " fixedBits=" + fixedBits + " numNonZ=" + numNonZero);
+        byte data[] = variance.toByteArray();
+        T key = makeKey(data);
+        byte[] hash = DataHelper.xor(key.getData(), _us.getData());
+        T rv = makeKey(hash);
+
+        // DEBUG
+        //int range = getRange(rv);
+        //if (range < obegin || range > oend) {
+        //    throw new IllegalStateException("Generate random key failed range=" + range + " for " + rv + " meant for bucket " + bucket);
+        //}
+
+        return rv;
+    }
+    
+    /**
+     *  Make a new SimpleDataStrucure from the data
+     *  @param data size <= SDS length, else throws IAE
+     *              Can be 1 bigger if top byte is zero
+     */
+    private T makeKey(byte[] data) {
+        int len = _us.length();
+        int dlen = data.length;
+        if (dlen > len + 1 ||
+            (dlen == len + 1 && data[0] != 0))
+            throw new IllegalArgumentException("bad length " + dlen + " > " + len);
+        T rv;
+        try {
+            rv = (T) _us.getClass().newInstance();
+        } catch (Exception e) {
+            _log.error("fail", e);
+            throw new RuntimeException(e);
+        }
+        if (dlen == len) {
+            rv.setData(data);
+        } else {
+            byte[] ndata = new byte[len];
+            if (dlen == len + 1) {
+                // one bigger
+                System.arraycopy(data, 1, ndata, 0, len);
+            } else {
+                // smaller
+                System.arraycopy(data, 0, ndata, len - dlen, dlen);
+            }
+            rv.setData(ndata);
+        }
+        return rv;
+    }
+
+    private static class Range<T extends SimpleDataStructure> {
+        private final int _bValue;
+        private final BigInteger _bigUs;
+        private final Map<T, Integer> _distanceCache;
+
+        public Range(T us, int bValue) {
+            _bValue = bValue;
+            _bigUs = new BigInteger(1, us.getData());
+            _distanceCache = new LHMCache(256);
+        }
+
+        /** @return 0 to max-1 or -1 for us */
+        public int getRange(T key) {
+            Integer rv;
+            synchronized (_distanceCache) {
+                rv = _distanceCache.get(key);
+                if (rv == null) {
+                    // easy way when _bValue == 1
+                    //rv = Integer.valueOf(_bigUs.xor(new BigInteger(1, key.getData())).bitLength() - 1);
+                    BigInteger xor = _bigUs.xor(new BigInteger(1, key.getData()));
+                    int range = xor.bitLength() - 1;
+                    if (_bValue > 1) {
+                        int toShift = range + 1 - _bValue;
+                        int highbit = range;
+                        range <<= _bValue - 1;
+                        if (toShift >= 0) {
+                            int extra = xor.clearBit(highbit).shiftRight(toShift).intValue();
+                            range += extra;
+                            //Log log = I2PAppContext.getGlobalContext().logManager().getLog(KBucketSet.class);
+                            //if (log.shouldLog(Log.DEBUG))
+                            //    log.debug("highbit " + highbit + " toshift " + toShift + " extra " + extra + " new " + range);
+                        }
+                    }
+                    rv = Integer.valueOf(range);
+                    _distanceCache.put(key, rv);
+                }
+            }
+            return rv.intValue();
+        }
+
+        public void clear() {
+            synchronized (_distanceCache) {
+                _distanceCache.clear();
+            }
+        }
+    }
+
+    /**
+     *  For Collections.binarySearch.
+     *  getRangeBegin == getRangeEnd.
+     */
+    private static class DummyBucket<T extends SimpleDataStructure> implements KBucket<T> {
+        private final int r;
+
+        public DummyBucket(int range) {
+            r = range;
+        }
+
+        public int getRangeBegin() { return r; }
+        public int getRangeEnd() { return r; }
+
+        public int getKeyCount() {
+            return 0;
+        }
+
+        public Set<T> getEntries() {
+            throw new UnsupportedOperationException();
+        }
+
+        public void getEntries(SelectionCollector<T> collector) {
+            throw new UnsupportedOperationException();
+        }
+    
+        public void clear() {}
+
+        public boolean add(T peer) {
+            throw new UnsupportedOperationException();
+        }
+    
+        public boolean remove(T peer) {
+            return false;
+        }
+
+        public void setLastChanged() {}
+
+        public long getLastChanged() {
+            return 0;
+        }
+    }
+
+    /**
+     *  For Collections.binarySearch.
+     *  Returns equal for any overlap.
+     */
+    private static class BucketComparator implements Comparator<KBucket> {
+        public int compare(KBucket l, KBucket r) {
+            if (l.getRangeEnd() < r.getRangeBegin())
+                return -1;
+            if (l.getRangeBegin() > r.getRangeEnd())
+                return 1;
+            return 0;
+        }
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder buf = new StringBuilder(1024);
+        buf.append("Bucket set rooted on: ").append(_us.toString())
+           .append(" K=").append(BUCKET_SIZE)
+           .append(" B=").append(B_VALUE)
+           .append(" with ").append(size())
+           .append(" keys in ").append(_buckets.size()).append(" buckets:\n");
+        getReadLock();
+        try {
+            int len = _buckets.size();
+            for (int i = 0; i < len; i++) {
+                KBucket b = _buckets.get(i);
+                buf.append("* Bucket ").append(i).append("/").append(len).append(": ");
+                buf.append(b.toString()).append("\n");
+            }
+        } finally { releaseReadLock(); }
+        return buf.toString();
+    }
+}

apps/i2psnark/java/src/net/i2p/kademlia/KBucketTrimmer.java

@@ -0,0 +1,20 @@
+package net.i2p.kademlia;
+
+import net.i2p.data.SimpleDataStructure;
+
+/**
+ *  Called when a kbucket can no longer be split and is too big
+ *  @since 0.9.2
+ */
+public interface KBucketTrimmer<K extends SimpleDataStructure> {
+    /**
+     *  Called from add() just before adding the entry.
+     *  You may call getEntries() and/or remove() from here.
+     *  Do NOT call add().
+     *  To always discard a newer entry, always return false.
+     *
+     *  @param kbucket the kbucket that is now too big
+     *  @return true to actually add the entry.
+     */
+    public boolean trim(KBucket<K> kbucket, K toAdd);
+}

apps/i2psnark/java/src/net/i2p/kademlia/RandomIfOldTrimmer.java

@@ -0,0 +1,21 @@
+package net.i2p.kademlia;
+
+import net.i2p.I2PAppContext;
+import net.i2p.data.SimpleDataStructure;
+
+/**
+ *  Removes a random element, but only if the bucket hasn't changed in 5 minutes.
+ *  @since 0.9.2
+ */
+public class RandomIfOldTrimmer<T extends SimpleDataStructure> extends RandomTrimmer<T> {
+
+    public RandomIfOldTrimmer(I2PAppContext ctx, int max) {
+        super(ctx, max);
+    }
+
+    public boolean trim(KBucket<T> kbucket, T toAdd) {
+        if (kbucket.getLastChanged() > _ctx.clock().now() - 5*60*1000)
+            return false;
+        return super.trim(kbucket, toAdd);
+    }
+}

apps/i2psnark/java/src/net/i2p/kademlia/RandomTrimmer.java

@@ -0,0 +1,31 @@
+package net.i2p.kademlia;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import net.i2p.I2PAppContext;
+import net.i2p.data.SimpleDataStructure;
+
+/**
+ *  Removes a random element. Not resistant to flooding.
+ *  @since 0.9.2
+ */
+public class RandomTrimmer<T extends SimpleDataStructure> implements KBucketTrimmer<T> {
+    protected final I2PAppContext _ctx;
+    private final int _max;
+
+    public RandomTrimmer(I2PAppContext ctx, int max) {
+        _ctx = ctx;
+        _max = max;
+    }
+
+    public boolean trim(KBucket<T> kbucket, T toAdd) {
+        List<T> e = new ArrayList(kbucket.getEntries());
+        int sz = e.size();
+        // concurrency
+        if (sz < _max)
+            return true;
+        T toRemove = e.get(_ctx.random().nextInt(sz));
+        return kbucket.remove(toRemove);
+    }
+}

apps/i2psnark/java/src/net/i2p/kademlia/RejectTrimmer.java

@@ -0,0 +1,13 @@
+package net.i2p.kademlia;
+
+import net.i2p.data.SimpleDataStructure;
+
+/**
+ *  Removes nothing and always rejects the add. Flood resistant..
+ *  @since 0.9.2
+ */
+public class RejectTrimmer<T extends SimpleDataStructure> implements KBucketTrimmer<T> {
+    public boolean trim(KBucket<T> kbucket, T toAdd) {
+        return false;
+    }
+}

apps/i2psnark/java/src/net/i2p/kademlia/SelectionCollector.java

@@ -0,0 +1,11 @@
+package net.i2p.kademlia;
+
+import net.i2p.data.SimpleDataStructure;
+
+/**
+ * Visit kbuckets, gathering matches
+ * @since 0.9.2
+ */
+public interface SelectionCollector<T extends SimpleDataStructure> {
+    public void add(T entry);
+}

apps/i2psnark/java/src/net/i2p/kademlia/XORComparator.java

@@ -0,0 +1,28 @@
+package net.i2p.kademlia;
+
+import java.util.Comparator;
+
+import net.i2p.data.DataHelper;
+import net.i2p.data.SimpleDataStructure;
+
+/**
+ * Help sort Hashes in relation to a base key using the XOR metric
+ *
+ * @since 0.9.2
+ */
+class XORComparator<T extends SimpleDataStructure> implements Comparator<T> {
+    private final byte[] _base;
+
+    /**
+     * @param target key to compare distances with
+     */
+    public XORComparator(T target) {
+        _base = target.getData();
+    }
+
+    public int compare(T lhs, T rhs) {
+        byte lhsDelta[] = DataHelper.xor(lhs.getData(), _base);
+        byte rhsDelta[] = DataHelper.xor(rhs.getData(), _base);
+        return DataHelper.compareTo(lhsDelta, rhsDelta);
+    }
+}

apps/i2psnark/java/src/net/i2p/kademlia/package.html

@@ -0,0 +1,6 @@
+<html><body><p>
+This is a major rewrite of KBucket, KBucketSet, and KBucketImpl from net.i2p.router.networkdb.kademlia.
+The classes are now generic to support SHA1. SHA256, or other key lengths.
+The long-term goal is to prove out this new implementation in i2psnark,
+then move it to core, then convert the network database to use it.
+</p></body></html>

apps/i2psnark/java/src/org/klomp/snark/BitField.java

@@ -39,7 +39,6 @@ public class BitField
     this.size = size;
     int arraysize = ((size-1)/8)+1;
     bitfield = new byte[arraysize];
-    this.count = 0;
   }
 
   /**
@@ -60,7 +59,6 @@ public class BitField
     // cleared or clear them explicitly ourselves.
     System.arraycopy(bitfield, 0, this.bitfield, 0, arraysize);
 
-    this.count = 0;
     for (int i = 0; i < size; i++)
       if (get(i))
         this.count++;
@@ -99,9 +97,11 @@ public class BitField
       throw new IndexOutOfBoundsException(Integer.toString(bit));
     int index = bit/8;
     int mask = 128 >> (bit % 8);
-    if ((bitfield[index] & mask) == 0) {
-      count++;
-      bitfield[index] |= mask;
+    synchronized(this) {
+        if ((bitfield[index] & mask) == 0) {
+            count++;
+            bitfield[index] |= mask;
+        }
     }
   }
 

apps/i2psnark/java/src/org/klomp/snark/CompleteListener.java

@@ -0,0 +1,60 @@
+/* CompleteListener - Callback for Snark events
+   
+   Copyright (C) 2003 Mark J. Wielaard
+
+   This file is part of Snark.
+   
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+ 
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+ 
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software Foundation,
+   Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+*/
+
+package org.klomp.snark;
+
+/**
+ *  Callback for Snark events.
+ *  @since 0.9.4 moved from Snark.java
+ */
+public interface CompleteListener {
+    public void torrentComplete(Snark snark);
+    public void updateStatus(Snark snark);
+
+    /**
+     * We transitioned from magnet mode, we have now initialized our
+     * metainfo and storage. The listener should now call getMetaInfo()
+     * and save the data to disk.
+     *
+     * @return the new name for the torrent or null on error
+     * @since 0.8.4
+     */
+    public String gotMetaInfo(Snark snark);
+
+    /**
+     * @since 0.9
+     */
+    public void fatal(Snark snark, String error);
+
+    /**
+     * @since 0.9.2
+     */
+    public void addMessage(Snark snark, String message);
+
+    /**
+     * @since 0.9.4
+     */
+    public void gotPiece(Snark snark);
+
+    // not really listeners but the easiest way to get back to an optional SnarkManager
+    public long getSavedTorrentTime(Snark snark);
+    public BitField getSavedTorrentBitField(Snark snark);
+}

apps/i2psnark/java/src/org/klomp/snark/ConnectionAcceptor.java

@@ -29,8 +29,12 @@ import net.i2p.I2PAppContext;
 import net.i2p.I2PException;
 import net.i2p.client.streaming.I2PServerSocket;
 import net.i2p.client.streaming.I2PSocket;
+import net.i2p.data.Hash;
 import net.i2p.util.I2PAppThread;
 import net.i2p.util.Log;
+import net.i2p.util.ObjectCounter;
+import net.i2p.util.SimpleScheduler;
+import net.i2p.util.SimpleTimer;
 
 /**
  * Accepts connections on a TCP port and routes them to sub-acceptors.
@@ -41,11 +45,15 @@ public class ConnectionAcceptor implements Runnable
   private I2PServerSocket serverSocket;
   private PeerAcceptor peeracceptor;
   private Thread thread;
-  private I2PSnarkUtil _util;
+  private final I2PSnarkUtil _util;
+  private final ObjectCounter<Hash> _badCounter = new ObjectCounter();
 
   private boolean stop;
   private boolean socketChanged;
 
+  private static final int MAX_BAD = 2;
+  private static final long BAD_CLEAN_INTERVAL = 30*60*1000;
+
   public ConnectionAcceptor(I2PSnarkUtil util) { _util = util; }
   
   public synchronized void startAccepting(PeerCoordinatorSet set, I2PServerSocket socket) {
@@ -59,6 +67,7 @@ public class ConnectionAcceptor implements Runnable
           thread = new I2PAppThread(this, "I2PSnark acceptor");
           thread.setDaemon(true);
           thread.start();
+          _util.getContext().simpleScheduler().addPeriodicEvent(new Cleaner(), BAD_CLEAN_INTERVAL);
       }
     }
   }
@@ -70,11 +79,10 @@ public class ConnectionAcceptor implements Runnable
     this.peeracceptor = peeracceptor;
     _util = util;
     
-    socketChanged = false;
-    stop = false;
     thread = new I2PAppThread(this, "I2PSnark acceptor");
     thread.setDaemon(true);
     thread.start();
+    _util.getContext().simpleScheduler().addPeriodicEvent(new Cleaner(), BAD_CLEAN_INTERVAL);
   }
 
   public void halt()
@@ -138,7 +146,13 @@ public class ConnectionAcceptor implements Runnable
                 }
             } else {
                 if (socket.getPeerDestination().equals(_util.getMyDestination())) {
-                    _util.debug("Incoming connection from myself", Snark.ERROR);
+                    _log.error("Incoming connection from myself");
+                    try { socket.close(); } catch (IOException ioe) {}
+                    continue;
+                }
+                if (_badCounter.count(socket.getPeerDestination().calculateHash()) >= MAX_BAD) {
+                    if (_log.shouldLog(Log.WARN))
+                        _log.warn("Rejecting connection from " + socket.getPeerDestination().calculateHash() + " after " + MAX_BAD + " failures");
                     try { socket.close(); } catch (IOException ioe) {}
                     continue;
                 }
@@ -149,13 +163,13 @@ public class ConnectionAcceptor implements Runnable
         catch (I2PException ioe)
           {
             if (!socketChanged) {
-                _util.debug("Error while accepting: " + ioe, Snark.ERROR);
+                _log.error("Error while accepting", ioe);
                 stop = true;
             }
           }
         catch (IOException ioe)
           {
-            _util.debug("Error while accepting: " + ioe, Snark.ERROR);
+            _log.error("Error while accepting", ioe);
             stop = true;
           }
         // catch oom?
@@ -171,10 +185,12 @@ public class ConnectionAcceptor implements Runnable
   }
   
   private class Handler implements Runnable {
-      private I2PSocket _socket;
+      private final I2PSocket _socket;
+
       public Handler(I2PSocket socket) {
           _socket = socket;
       }
+
       public void run() {
           try {
               InputStream in = _socket.getInputStream();
@@ -182,13 +198,23 @@ public class ConnectionAcceptor implements Runnable
               // this is for the readahead in PeerAcceptor.connection()
               in = new BufferedInputStream(in);
               if (_log.shouldLog(Log.DEBUG))
-                  _log.debug("Handling socket from " + _socket.getPeerDestination().calculateHash().toBase64());
+                  _log.debug("Handling socket from " + _socket.getPeerDestination().calculateHash());
               peeracceptor.connection(_socket, in, out);
+          } catch (PeerAcceptor.ProtocolException ihe) {
+              _badCounter.increment(_socket.getPeerDestination().calculateHash());
+              if (_log.shouldLog(Log.INFO))
+                  _log.info("Protocol error from " + _socket.getPeerDestination().calculateHash(), ihe);
+              try { _socket.close(); } catch (IOException ignored) { }
           } catch (IOException ioe) {
               if (_log.shouldLog(Log.DEBUG))
-                  _log.debug("Error handling connection from " + _socket.getPeerDestination().calculateHash().toBase64(), ioe);
+                  _log.debug("Error handling connection from " + _socket.getPeerDestination().calculateHash(), ioe);
               try { _socket.close(); } catch (IOException ignored) { }
           }
       }
   }
+
+    /** @since 0.9.1 */    
+    private class Cleaner implements SimpleTimer.TimedEvent {
+        public void timeReached() { _badCounter.clear(); }
+    }
 }

apps/i2psnark/java/src/org/klomp/snark/CoordinatorListener.java

@@ -24,7 +24,7 @@ package org.klomp.snark;
 /**
  * Callback used when some peer changes state.
  */
-public interface CoordinatorListener
+interface CoordinatorListener
 {
   /**
    * Called when the PeerCoordinator notices a change in the state of a peer.
@@ -40,4 +40,5 @@ public interface CoordinatorListener
   public boolean overUploadLimit(int uploaders);
   public boolean overUpBWLimit();
   public boolean overUpBWLimit(long total);
+  public void addMessage(String message);
 }

apps/i2psnark/java/src/org/klomp/snark/DataLoader.java

@@ -1,5 +1,7 @@
 package org.klomp.snark;
 
+import net.i2p.data.ByteArray;
+
 /**
  * Callback used to fetch data
  * @since 0.8.2
@@ -10,5 +12,5 @@ interface DataLoader
    *  This is the callback that PeerConnectionOut calls to get the data from disk
    *  @return bytes or null for errors
    */
-    public byte[] loadData(int piece, int begin, int length);
+    public ByteArray loadData(int piece, int begin, int length);
 }

apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java

@@ -1,7 +1,6 @@
 package org.klomp.snark;
 
 import java.io.ByteArrayInputStream;
-import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -15,7 +14,6 @@ import net.i2p.util.Log;
 import org.klomp.snark.bencode.BDecoder;
 import org.klomp.snark.bencode.BEncoder;
 import org.klomp.snark.bencode.BEValue;
-import org.klomp.snark.bencode.InvalidBEncodingException;
 
 /**
  * REF: BEP 10 Extension Protocol
@@ -24,14 +22,15 @@ import org.klomp.snark.bencode.InvalidBEncodingException;
  */
 abstract class ExtensionHandler {
 
-    private static final Log _log = I2PAppContext.getGlobalContext().logManager().getLog(ExtensionHandler.class);
-
     public static final int ID_HANDSHAKE = 0;
     public static final int ID_METADATA = 1;
     public static final String TYPE_METADATA = "ut_metadata";
     public static final int ID_PEX = 2;
     /** not ut_pex since the compact format is different */
     public static final String TYPE_PEX = "i2p_pex";
+    public static final int ID_DHT = 3;
+    /** not using the option bit since the compact format is different */
+    public static final String TYPE_DHT = "i2p_dht";
     /** Pieces * SHA1 Hash length, + 25% extra for file names, benconding overhead, etc */
     private static final int MAX_METADATA_SIZE = Storage.MAX_PIECES * 20 * 5 / 4;
     private static final int PARALLEL_REQUESTS = 3;
@@ -39,15 +38,23 @@ abstract class ExtensionHandler {
 
   /**
    *  @param metasize -1 if unknown
+   *  @param pexAndMetadata advertise these capabilities
+   *  @param dht advertise DHT capability
    *  @return bencoded outgoing handshake message
    */
-    public static byte[] getHandshake(int metasize) {
+    public static byte[] getHandshake(int metasize, boolean pexAndMetadata, boolean dht) {
         Map<String, Object> handshake = new HashMap();
         Map<String, Integer> m = new HashMap();
-        m.put(TYPE_METADATA, Integer.valueOf(ID_METADATA));
-        m.put(TYPE_PEX, Integer.valueOf(ID_PEX));
-        if (metasize >= 0)
-            handshake.put("metadata_size", Integer.valueOf(metasize));
+        if (pexAndMetadata) {
+            m.put(TYPE_METADATA, Integer.valueOf(ID_METADATA));
+            m.put(TYPE_PEX, Integer.valueOf(ID_PEX));
+            if (metasize >= 0)
+                handshake.put("metadata_size", Integer.valueOf(metasize));
+        }
+        if (dht) {
+            m.put(TYPE_DHT, Integer.valueOf(ID_DHT));
+        }
+        // include the map even if empty so the far-end doesn't NPE
         handshake.put("m", m);
         handshake.put("p", Integer.valueOf(6881));
         handshake.put("v", "I2PSnark");
@@ -56,21 +63,24 @@ abstract class ExtensionHandler {
     }
 
     public static void handleMessage(Peer peer, PeerListener listener, int id, byte[] bs) {
-        if (_log.shouldLog(Log.INFO))
-            _log.info("Got extension msg " + id + " length " + bs.length + " from " + peer);
+        Log log = I2PAppContext.getGlobalContext().logManager().getLog(ExtensionHandler.class);
+        if (log.shouldLog(Log.INFO))
+            log.info("Got extension msg " + id + " length " + bs.length + " from " + peer);
         if (id == ID_HANDSHAKE)
-            handleHandshake(peer, listener, bs);
+            handleHandshake(peer, listener, bs, log);
         else if (id == ID_METADATA)
-            handleMetadata(peer, listener, bs);
+            handleMetadata(peer, listener, bs, log);
         else if (id == ID_PEX)
-            handlePEX(peer, listener, bs);
-        else if (_log.shouldLog(Log.INFO))
-            _log.info("Unknown extension msg " + id + " from " + peer);
+            handlePEX(peer, listener, bs, log);
+        else if (id == ID_DHT)
+            handleDHT(peer, listener, bs, log);
+        else if (log.shouldLog(Log.INFO))
+            log.info("Unknown extension msg " + id + " from " + peer);
     }
 
-    private static void handleHandshake(Peer peer, PeerListener listener, byte[] bs) {
-        if (_log.shouldLog(Log.DEBUG))
-            _log.debug("Got handshake msg from " + peer);
+    private static void handleHandshake(Peer peer, PeerListener listener, byte[] bs, Log log) {
+        if (log.shouldLog(Log.DEBUG))
+            log.debug("Got handshake msg from " + peer);
         try {
             // this throws NPE on missing keys
             InputStream is = new ByteArrayInputStream(bs);
@@ -81,20 +91,26 @@ abstract class ExtensionHandler {
             Map<String, BEValue> msgmap = map.get("m").getMap();
 
             if (msgmap.get(TYPE_PEX) != null) {
-                if (_log.shouldLog(Log.WARN))
-                    _log.debug("Peer supports PEX extension: " + peer);
+                if (log.shouldLog(Log.DEBUG))
+                    log.debug("Peer supports PEX extension: " + peer);
                 // peer state calls peer listener calls sendPEX()
             }
 
+            if (msgmap.get(TYPE_DHT) != null) {
+                if (log.shouldLog(Log.DEBUG))
+                    log.debug("Peer supports DHT extension: " + peer);
+                // peer state calls peer listener calls sendDHT()
+            }
+
             MagnetState state = peer.getMagnetState();
 
             if (msgmap.get(TYPE_METADATA) == null) {
-                if (_log.shouldLog(Log.WARN))
-                    _log.debug("Peer does not support metadata extension: " + peer);
+                if (log.shouldLog(Log.DEBUG))
+                    log.debug("Peer does not support metadata extension: " + peer);
                 // drop if we need metainfo and we haven't found anybody yet
                 synchronized(state) {
                     if (!state.isInitialized()) {
-                        _log.debug("Dropping peer, we need metadata! " + peer);
+                        log.debug("Dropping peer, we need metadata! " + peer);
                         peer.disconnect();
                     }
                 }
@@ -103,20 +119,20 @@ abstract class ExtensionHandler {
 
             BEValue msize = map.get("metadata_size");
             if (msize == null) {
-                if (_log.shouldLog(Log.WARN))
-                    _log.debug("Peer does not have the metainfo size yet: " + peer);
+                if (log.shouldLog(Log.DEBUG))
+                    log.debug("Peer does not have the metainfo size yet: " + peer);
                 // drop if we need metainfo and we haven't found anybody yet
                 synchronized(state) {
                     if (!state.isInitialized()) {
-                        _log.debug("Dropping peer, we need metadata! " + peer);
+                        log.debug("Dropping peer, we need metadata! " + peer);
                         peer.disconnect();
                     }
                 }
                 return;
             }
             int metaSize = msize.getInt();
-            if (_log.shouldLog(Log.WARN))
-                _log.debug("Got the metainfo size: " + metaSize);
+            if (log.shouldLog(Log.DEBUG))
+                log.debug("Got the metainfo size: " + metaSize);
 
             int remaining;
             synchronized(state) {
@@ -125,21 +141,21 @@ abstract class ExtensionHandler {
 
                 if (state.isInitialized()) {
                     if (state.getSize() != metaSize) {
-                        if (_log.shouldLog(Log.WARN))
-                            _log.debug("Wrong metainfo size " + metaSize + " from: " + peer);
+                        if (log.shouldLog(Log.DEBUG))
+                            log.debug("Wrong metainfo size " + metaSize + " from: " + peer);
                         peer.disconnect();
                         return;
                     }
                 } else {
                     // initialize it
                     if (metaSize > MAX_METADATA_SIZE) {
-                        if (_log.shouldLog(Log.WARN))
-                            _log.debug("Huge metainfo size " + metaSize + " from: " + peer);
+                        if (log.shouldLog(Log.DEBUG))
+                            log.debug("Huge metainfo size " + metaSize + " from: " + peer);
                         peer.disconnect(false);
                         return;
                     }
-                    if (_log.shouldLog(Log.INFO))
-                        _log.info("Initialized state, metadata size = " + metaSize + " from " + peer);
+                    if (log.shouldLog(Log.INFO))
+                        log.info("Initialized state, metadata size = " + metaSize + " from " + peer);
                     state.initialize(metaSize);
                 }
                 remaining = state.chunksRemaining();
@@ -152,13 +168,13 @@ abstract class ExtensionHandler {
                 synchronized(state) {
                     chk = state.getNextRequest();
                 }
-                if (_log.shouldLog(Log.INFO))
-                    _log.info("Request chunk " + chk + " from " + peer);
+                if (log.shouldLog(Log.INFO))
+                    log.info("Request chunk " + chk + " from " + peer);
                 sendRequest(peer, chk);
             }
         } catch (Exception e) {
-            if (_log.shouldLog(Log.WARN))
-                _log.warn("Handshake exception from " + peer, e);
+            if (log.shouldLog(Log.WARN))
+                log.warn("Handshake exception from " + peer, e);
         }
     }
 
@@ -166,15 +182,13 @@ abstract class ExtensionHandler {
     private static final int TYPE_DATA = 1;
     private static final int TYPE_REJECT = 2;
 
-    private static final int CHUNK_SIZE = 16*1024;
-
     /**
      * REF: BEP 9
      * @since 0.8.4
      */
-    private static void handleMetadata(Peer peer, PeerListener listener, byte[] bs) {
-        if (_log.shouldLog(Log.DEBUG))
-            _log.debug("Got metadata msg from " + peer);
+    private static void handleMetadata(Peer peer, PeerListener listener, byte[] bs, Log log) {
+        if (log.shouldLog(Log.DEBUG))
+            log.debug("Got metadata msg from " + peer);
         try {
             InputStream is = new ByteArrayInputStream(bs);
             BDecoder dec = new BDecoder(is);
@@ -185,8 +199,8 @@ abstract class ExtensionHandler {
 
             MagnetState state = peer.getMagnetState();
             if (type == TYPE_REQUEST) {
-                if (_log.shouldLog(Log.DEBUG))
-                    _log.debug("Got request for " + piece + " from: " + peer);
+                if (log.shouldLog(Log.DEBUG))
+                    log.debug("Got request for " + piece + " from: " + peer);
                 byte[] pc;
                 synchronized(state) {
                     pc = state.getChunk(piece);
@@ -197,8 +211,8 @@ abstract class ExtensionHandler {
                 listener.uploaded(peer, pc.length);
             } else if (type == TYPE_DATA) {
                 int size = map.get("total_size").getInt();
-                if (_log.shouldLog(Log.DEBUG))
-                    _log.debug("Got data for " + piece + " length " + size + " from: " + peer);
+                if (log.shouldLog(Log.DEBUG))
+                    log.debug("Got data for " + piece + " length " + size + " from: " + peer);
                 boolean done;
                 int chk = -1;
                 synchronized(state) {
@@ -207,14 +221,14 @@ abstract class ExtensionHandler {
                     int len = is.available();
                     if (len != size) {
                         // probably fatal
-                        if (_log.shouldLog(Log.WARN))
-                            _log.warn("total_size " + size + " but avail data " + len);
+                        if (log.shouldLog(Log.WARN))
+                            log.warn("total_size " + size + " but avail data " + len);
                     }
                     peer.downloaded(len);
                     listener.downloaded(peer, len);
                     done = state.saveChunk(piece, bs, bs.length - len, len);
-                    if (_log.shouldLog(Log.INFO))
-                        _log.info("Got chunk " + piece + " from " + peer);
+                    if (log.shouldLog(Log.INFO))
+                        log.info("Got chunk " + piece + " from " + peer);
                     if (!done)
                         chk = state.getNextRequest();
                 }
@@ -223,26 +237,26 @@ abstract class ExtensionHandler {
                     // Done!
                     // PeerState will call the listener (peer coord), who will
                     // check to see if the MagnetState has it
-                    if (_log.shouldLog(Log.WARN))
-                        _log.warn("Got last chunk from " + peer);
+                    if (log.shouldLog(Log.WARN))
+                        log.warn("Got last chunk from " + peer);
                 } else {
                     // get the next chunk
-                    if (_log.shouldLog(Log.INFO))
-                        _log.info("Request chunk " + chk + " from " + peer);
+                    if (log.shouldLog(Log.INFO))
+                        log.info("Request chunk " + chk + " from " + peer);
                     sendRequest(peer, chk);
                 }
             } else if (type == TYPE_REJECT) {
-                if (_log.shouldLog(Log.WARN))
-                    _log.warn("Got reject msg from " + peer);
+                if (log.shouldLog(Log.WARN))
+                    log.warn("Got reject msg from " + peer);
                 peer.disconnect(false);
             } else {
-                if (_log.shouldLog(Log.WARN))
-                    _log.warn("Got unknown metadata msg from " + peer);
+                if (log.shouldLog(Log.WARN))
+                    log.warn("Got unknown metadata msg from " + peer);
                 peer.disconnect(false);
             }
         } catch (Exception e) {
-            if (_log.shouldLog(Log.WARN))
-                _log.info("Metadata ext. msg. exception from " + peer, e);
+            if (log.shouldLog(Log.INFO))
+                log.info("Metadata ext. msg. exception from " + peer, e);
             // fatal ?
             peer.disconnect(false);
         }
@@ -252,9 +266,11 @@ abstract class ExtensionHandler {
         sendMessage(peer, TYPE_REQUEST, piece);
     }
 
+  /****
     private static void sendReject(Peer peer, int piece) {
         sendMessage(peer, TYPE_REJECT, piece);
     }
+  ****/
 
     /** REQUEST and REJECT are the same except for message type */
     private static void sendMessage(Peer peer, int type, int piece) {
@@ -267,8 +283,8 @@ abstract class ExtensionHandler {
             peer.sendExtension(hisMsgCode, payload);
         } catch (Exception e) {
             // NPE, no metadata capability
-            if (_log.shouldLog(Log.WARN))
-                _log.info("Metadata send req msg exception to " + peer, e);
+            //if (log.shouldLog(Log.INFO))
+            //    log.info("Metadata send req msg exception to " + peer, e);
         }
     }
 
@@ -286,8 +302,8 @@ abstract class ExtensionHandler {
             peer.sendExtension(hisMsgCode, payload);
         } catch (Exception e) {
             // NPE, no metadata caps
-            if (_log.shouldLog(Log.WARN))
-                _log.info("Metadata send piece msg exception to " + peer, e);
+            //if (log.shouldLog(Log.INFO))
+            //    log.info("Metadata send piece msg exception to " + peer, e);
         }
     }
 
@@ -301,15 +317,18 @@ abstract class ExtensionHandler {
      * added.f and dropped unsupported
      * @since 0.8.4
      */
-    private static void handlePEX(Peer peer, PeerListener listener, byte[] bs) {
-        if (_log.shouldLog(Log.DEBUG))
-            _log.debug("Got PEX msg from " + peer);
+    private static void handlePEX(Peer peer, PeerListener listener, byte[] bs, Log log) {
+        if (log.shouldLog(Log.DEBUG))
+            log.debug("Got PEX msg from " + peer);
         try {
             InputStream is = new ByteArrayInputStream(bs);
             BDecoder dec = new BDecoder(is);
             BEValue bev = dec.bdecodeMap();
             Map<String, BEValue> map = bev.getMap();
-            byte[] ids = map.get("added").getBytes();
+            bev = map.get("added");
+            if (bev == null)
+                return;
+            byte[] ids = bev.getBytes();
             if (ids.length < HASH_LENGTH)
                 return;
             int len = Math.min(ids.length, (I2PSnarkUtil.MAX_CONNECTIONS - 1) * HASH_LENGTH);
@@ -319,14 +338,36 @@ abstract class ExtensionHandler {
                 System.arraycopy(ids, off, hash, 0, HASH_LENGTH);
                 if (DataHelper.eq(hash, peer.getPeerID().getDestHash()))
                     continue;
-                PeerID pID = new PeerID(hash);
+                PeerID pID = new PeerID(hash, listener.getUtil());
                 peers.add(pID);
             }
             // could include ourselves, listener must remove
             listener.gotPeers(peer, peers);
         } catch (Exception e) {
-            if (_log.shouldLog(Log.WARN))
-                _log.info("PEX msg exception from " + peer, e);
+            if (log.shouldLog(Log.INFO))
+                log.info("PEX msg exception from " + peer, e);
+            //peer.disconnect(false);
+        }
+    }
+
+    /**
+     * Receive the DHT port numbers
+     * @since DHT
+     */
+    private static void handleDHT(Peer peer, PeerListener listener, byte[] bs, Log log) {
+        if (log.shouldLog(Log.DEBUG))
+            log.debug("Got DHT msg from " + peer);
+        try {
+            InputStream is = new ByteArrayInputStream(bs);
+            BDecoder dec = new BDecoder(is);
+            BEValue bev = dec.bdecodeMap();
+            Map<String, BEValue> map = bev.getMap();
+            int qport = map.get("port").getInt();
+            int rport = map.get("rport").getInt();
+            listener.gotPort(peer, qport, rport);
+        } catch (Exception e) {
+            if (log.shouldLog(Log.INFO))
+                log.info("DHT msg exception from " + peer, e);
             //peer.disconnect(false);
         }
     }
@@ -353,9 +394,27 @@ abstract class ExtensionHandler {
             peer.sendExtension(hisMsgCode, payload);
         } catch (Exception e) {
             // NPE, no PEX caps
-            if (_log.shouldLog(Log.WARN))
-                _log.info("PEX msg exception to " + peer, e);
+            //if (log.shouldLog(Log.INFO))
+            //    log.info("PEX msg exception to " + peer, e);
         }
     }
 
+    /**
+     *  Send the DHT port numbers
+     *  @since DHT
+     */
+    public static void sendDHT(Peer peer, int qport, int rport) {
+        Map<String, Object> map = new HashMap();
+        map.put("port", Integer.valueOf(qport));
+        map.put("rport", Integer.valueOf(rport));
+        byte[] payload = BEncoder.bencode(map);
+        try {
+            int hisMsgCode = peer.getHandshakeMap().get("m").getMap().get(TYPE_DHT).getInt();
+            peer.sendExtension(hisMsgCode, payload);
+        } catch (Exception e) {
+            // NPE, no DHT caps
+            //if (log.shouldLog(Log.INFO))
+            //    log.info("DHT msg exception to " + peer, e);
+        }
+    }
 }

apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java

@@ -1,8 +1,10 @@
 package org.klomp.snark;
 
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -20,6 +22,7 @@ import net.i2p.client.streaming.I2PSocket;
 import net.i2p.client.streaming.I2PSocketEepGet;
 import net.i2p.client.streaming.I2PSocketManager;
 import net.i2p.client.streaming.I2PSocketManagerFactory;
+import net.i2p.client.streaming.I2PSocketOptions;
 import net.i2p.data.Base32;
 import net.i2p.data.DataFormatException;
 import net.i2p.data.Destination;
@@ -35,7 +38,7 @@ import net.i2p.util.SimpleTimer;
 import net.i2p.util.Translate;
 
 import org.klomp.snark.dht.DHT;
-//import org.klomp.snark.dht.KRPC;
+import org.klomp.snark.dht.KRPC;
 
 /**
  * I2P specific helpers for I2PSnark
@@ -53,26 +56,30 @@ public class I2PSnarkUtil {
     private String _i2cpHost;
     private int _i2cpPort;
     private final Map<String, String> _opts;
-    private I2PSocketManager _manager;
+    private volatile I2PSocketManager _manager;
     private boolean _configured;
-    private final Set<Hash> _shitlist;
+    private volatile boolean _connecting;
+    private final Set<Hash> _banlist;
     private int _maxUploaders;
     private int _maxUpBW;
     private int _maxConnections;
     private final File _tmpDir;
     private int _startupDelay;
     private boolean _shouldUseOT;
+    private boolean _shouldUseDHT;
     private boolean _areFilesPublic;
-    private String _openTrackerString;
+    private List<String> _openTrackers;
     private DHT _dht;
 
+    private static final int EEPGET_CONNECT_TIMEOUT = 45*1000;
+    private static final int EEPGET_CONNECT_TIMEOUT_SHORT = 5*1000;
     public static final int DEFAULT_STARTUP_DELAY = 3;
     public static final boolean DEFAULT_USE_OPENTRACKERS = true;
     public static final String DEFAULT_OPENTRACKERS = "http://tracker.welterde.i2p/a";
     public static final int DEFAULT_MAX_UP_BW = 8;  //KBps
     public static final int MAX_CONNECTIONS = 16; // per torrent
     public static final String PROP_MAX_BW = "i2cp.outboundBytesPerSecond";
-    //private static final boolean ENABLE_DHT = true;
+    public static final boolean DEFAULT_USE_DHT = true;
 
     public I2PSnarkUtil(I2PAppContext ctx) {
         _context = ctx;
@@ -80,13 +87,15 @@ public class I2PSnarkUtil {
         _opts = new HashMap();
         //setProxy("127.0.0.1", 4444);
         setI2CPConfig("127.0.0.1", 7654, null);
-        _shitlist = new ConcurrentHashSet();
-        _configured = false;
+        _banlist = new ConcurrentHashSet();
         _maxUploaders = Snark.MAX_TOTAL_UPLOADERS;
         _maxUpBW = DEFAULT_MAX_UP_BW;
         _maxConnections = MAX_CONNECTIONS;
         _startupDelay = DEFAULT_STARTUP_DELAY;
         _shouldUseOT = DEFAULT_USE_OPENTRACKERS;
+        // FIXME split if default has more than one
+        _openTrackers = Collections.singletonList(DEFAULT_OPENTRACKERS);
+        _shouldUseDHT = DEFAULT_USE_DHT;
         // This is used for both announce replies and .torrent file downloads,
         // so it must be available even if not connected to I2CP.
         // so much for multiple instances
@@ -115,6 +124,9 @@ public class I2PSnarkUtil {
     }
 ******/
     
+    /** @since 0.9.1 */
+    public I2PAppContext getContext() { return _context; }
+    
     public boolean configured() { return _configured; }
     
     public void setI2CPConfig(String i2cpHost, int i2cpPort, Map opts) {
@@ -184,11 +196,15 @@ public class I2PSnarkUtil {
     /** @since 0.8.9 */
     public void setFilesPublic(boolean yes) { _areFilesPublic = yes; }
 
+    /** @since 0.9.1 */
+    public File getTempDir() { return _tmpDir; }
+
     /**
      * Connect to the router, if we aren't already
      */
     synchronized public boolean connect() {
         if (_manager == null) {
+            _connecting = true;
             // try to find why reconnecting after stop
             if (_log.shouldLog(Log.DEBUG))
                 _log.debug("Connecting to I2P", new Exception("I did it"));
@@ -203,10 +219,14 @@ public class I2PSnarkUtil {
                 opts.setProperty("inbound.nickname", "I2PSnark");
             if (opts.getProperty("outbound.nickname") == null)
                 opts.setProperty("outbound.nickname", "I2PSnark");
+            if (opts.getProperty("outbound.priority") == null)
+                opts.setProperty("outbound.priority", "-10");
             // Dont do this for now, it is set in I2PSocketEepGet for announces,
             // we don't need fast handshake for peer connections.
             //if (opts.getProperty("i2p.streaming.connectDelay") == null)
             //    opts.setProperty("i2p.streaming.connectDelay", "500");
+            if (opts.getProperty(I2PSocketOptions.PROP_CONNECT_TIMEOUT) == null)
+                opts.setProperty(I2PSocketOptions.PROP_CONNECT_TIMEOUT, "75000");
             if (opts.getProperty("i2p.streaming.inactivityTimeout") == null)
                 opts.setProperty("i2p.streaming.inactivityTimeout", "240000");
             if (opts.getProperty("i2p.streaming.inactivityAction") == null)
@@ -219,11 +239,21 @@ public class I2PSnarkUtil {
             //    opts.setProperty("i2p.streaming.writeTimeout", "90000");
             //if (opts.getProperty("i2p.streaming.readTimeout") == null)
             //    opts.setProperty("i2p.streaming.readTimeout", "120000");
+            if (opts.getProperty("i2p.streaming.maxConnsPerMinute") == null)
+                opts.setProperty("i2p.streaming.maxConnsPerMinute", "2");
+            if (opts.getProperty("i2p.streaming.maxTotalConnsPerMinute") == null)
+                opts.setProperty("i2p.streaming.maxTotalConnsPerMinute", "8");
+            if (opts.getProperty("i2p.streaming.maxConnsPerHour") == null)
+                opts.setProperty("i2p.streaming.maxConnsPerHour", "20");
+            if (opts.getProperty("i2p.streaming.enforceProtocol") == null)
+                opts.setProperty("i2p.streaming.enforceProtocol", "true");
+            if (opts.getProperty("i2p.streaming.disableRejectLogging") == null)
+                opts.setProperty("i2p.streaming.disableRejectLogging", "true");
             _manager = I2PSocketManagerFactory.createManager(_i2cpHost, _i2cpPort, opts);
+            _connecting = false;
         }
-        // FIXME this only instantiates krpc once, left stuck with old manager
-        //if (ENABLE_DHT && _manager != null && _dht == null)
-        //    _dht = new KRPC(_context, _manager.getSession());
+        if (_shouldUseDHT && _manager != null && _dht == null)
+            _dht = new KRPC(_context, _manager.getSession());
         return (_manager != null);
     }
     
@@ -235,15 +265,35 @@ public class I2PSnarkUtil {
 
     public boolean connected() { return _manager != null; }
 
+    /** @since 0.9.1 */
+    public boolean isConnecting() { return _manager == null && _connecting; }
+
+    /**
+     *  For FetchAndAdd
+     *  @return null if not connected
+     *  @since 0.9.1
+     */
+    public I2PSocketManager getSocketManager() {
+        return _manager;
+    }
+
     /**
      * Destroy the destination itself
      */
-    public void disconnect() {
+    public synchronized void disconnect() {
+        if (_dht != null) {
+            _dht.stop();
+            _dht = null;
+        }
         I2PSocketManager mgr = _manager;
         // FIXME this can cause race NPEs elsewhere
         _manager = null;
-        _shitlist.clear();
-        mgr.destroySocketManager();
+        _banlist.clear();
+        if (mgr != null) {
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("Disconnecting from I2P", new Exception("I did it"));
+            mgr.destroySocketManager();
+        }
         // this will delete a .torrent file d/l in progress so don't do that...
         FileUtil.rmdir(_tmpDir, false);
         // in case the user will d/l a .torrent file next...
@@ -261,32 +311,45 @@ public class I2PSnarkUtil {
         if (addr.equals(getMyDestination()))
             throw new IOException("Attempt to connect to myself");
         Hash dest = addr.calculateHash();
-        if (_shitlist.contains(dest))
-            throw new IOException("Not trying to contact " + dest.toBase64() + ", as they are shitlisted");
+        if (_banlist.contains(dest))
+            throw new IOException("Not trying to contact " + dest.toBase64() + ", as they are banlisted");
         try {
             I2PSocket rv = _manager.connect(addr);
             if (rv != null)
-                _shitlist.remove(dest);
+                _banlist.remove(dest);
             return rv;
         } catch (I2PException ie) {
-            _shitlist.add(dest);
-            SimpleScheduler.getInstance().addEvent(new Unshitlist(dest), 10*60*1000);
+            _banlist.add(dest);
+            _context.simpleScheduler().addEvent(new Unbanlist(dest), 10*60*1000);
             throw new IOException("Unable to reach the peer " + peer + ": " + ie.getMessage());
         }
     }
     
-    private class Unshitlist implements SimpleTimer.TimedEvent {
+    private class Unbanlist implements SimpleTimer.TimedEvent {
         private Hash _dest;
-        public Unshitlist(Hash dest) { _dest = dest; }
-        public void timeReached() { _shitlist.remove(_dest); }
+        public Unbanlist(Hash dest) { _dest = dest; }
+        public void timeReached() { _banlist.remove(_dest); }
     }
     
     /**
-     * fetch the given URL, returning the file it is stored in, or null on error
+     * Fetch the given URL, returning the file it is stored in, or null on error.
+     * No retries.
      */
     public File get(String url) { return get(url, true, 0); }
+
+    /**
+     * @param rewrite if true, convert http://KEY.i2p/foo/announce to http://i2p/KEY/foo/announce
+     */
     public File get(String url, boolean rewrite) { return get(url, rewrite, 0); }
+
+    /**
+     * @param retries if < 0, set timeout to a few seconds
+     */
     public File get(String url, int retries) { return get(url, true, retries); }
+
+    /**
+     * @param retries if < 0, set timeout to a few seconds
+     */
     public File get(String url, boolean rewrite, int retries) {
         if (_log.shouldLog(Log.DEBUG))
             _log.debug("Fetching [" + url + "] proxy=" + _proxyHost + ":" + _proxyPort + ": " + _shouldProxy);
@@ -295,7 +358,7 @@ public class I2PSnarkUtil {
             // we could use the system tmp dir but deleteOnExit() doesn't seem to work on all platforms...
             out = SecureFile.createTempFile("i2psnark", null, _tmpDir);
         } catch (IOException ioe) {
-            ioe.printStackTrace();
+            _log.error("temp file error", ioe);
             if (out != null)
                 out.delete();
             return null;
@@ -307,12 +370,21 @@ public class I2PSnarkUtil {
         //_log.debug("Rewritten url [" + fetchURL + "]");
         //EepGet get = new EepGet(_context, _shouldProxy, _proxyHost, _proxyPort, retries, out.getAbsolutePath(), fetchURL);
         // Use our tunnel for announces and .torrent fetches too! Make sure we're connected first...
-        if (!connected()) {
-            if (!connect())
+        int timeout;
+        if (retries < 0) {
+            if (!connected())
                 return null;
+            timeout = EEPGET_CONNECT_TIMEOUT_SHORT;
+            retries = 0;
+        } else {
+            timeout = EEPGET_CONNECT_TIMEOUT;
+            if (!connected()) {
+                if (!connect())
+                    return null;
+            }
         }
         EepGet get = new I2PSocketEepGet(_context, _manager, retries, out.getAbsolutePath(), fetchURL);
-        if (get.fetch()) {
+        if (get.fetch(timeout)) {
             if (_log.shouldLog(Log.DEBUG))
                 _log.debug("Fetch successful [" + url + "]: size=" + out.length());
             return out;
@@ -324,6 +396,46 @@ public class I2PSnarkUtil {
         }
     }
     
+    /**
+     * Fetch to memory
+     * @param retries if < 0, set timeout to a few seconds
+     * @param initialSize buffer size
+     * @param maxSize fails if greater
+     * @return null on error
+     * @since 0.9.4
+     */
+    public byte[] get(String url, boolean rewrite, int retries, int initialSize, int maxSize) {
+        if (_log.shouldLog(Log.DEBUG))
+            _log.debug("Fetching [" + url + "] to memory");
+        String fetchURL = url;
+        if (rewrite)
+            fetchURL = rewriteAnnounce(url);
+        int timeout;
+        if (retries < 0) {
+            if (!connected())
+                return null;
+            timeout = EEPGET_CONNECT_TIMEOUT_SHORT;
+            retries = 0;
+        } else {
+            timeout = EEPGET_CONNECT_TIMEOUT;
+            if (!connected()) {
+                if (!connect())
+                    return null;
+            }
+        }
+        ByteArrayOutputStream out = new ByteArrayOutputStream(initialSize);
+        EepGet get = new I2PSocketEepGet(_context, _manager, retries, -1, maxSize, null, out, fetchURL);
+        if (get.fetch(timeout)) {
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("Fetch successful [" + url + "]: size=" + out.size());
+            return out.toByteArray();
+        } else {
+            if (_log.shouldLog(Log.WARN))
+                _log.warn("Fetch failed [" + url + "]");
+            return null;
+        }
+    }
+    
     public I2PServerSocket getServerSocket() { 
         I2PSocketManager mgr = _manager;
         if (mgr != null)
@@ -386,7 +498,8 @@ public class I2PSnarkUtil {
                     if (sess != null) {
                         byte[] b = Base32.decode(ip.substring(0, BASE32_HASH_LENGTH));
                         if (b != null) {
-                            Hash h = new Hash(b);
+                            //Hash h = new Hash(b);
+                            Hash h = Hash.create(b);
                             if (_log.shouldLog(Log.INFO))
                                 _log.info("Using existing session for lookup of " + ip);
                             try {
@@ -441,30 +554,26 @@ public class I2PSnarkUtil {
     }
     
     /** @param ot non-null */
-    public void setOpenTrackerString(String ot) { 
-        _openTrackerString = ot;
+    public void setOpenTrackers(List<String> ot) { 
+        _openTrackers = ot;
     }
 
-    public String getOpenTrackerString() { 
-        if (_openTrackerString == null)
-            return DEFAULT_OPENTRACKERS;
-        return _openTrackerString;
-    }
-
-    /** comma delimited list open trackers to use as backups */
-    /** sorted map of name to announceURL=baseURL */
+    /** List of open trackers to use as backups
+     *  @return non-null, possibly unmodifiable, empty if disabled
+     */
     public List<String> getOpenTrackers() { 
         if (!shouldUseOpenTrackers())
-            return null;
-        List<String> rv = new ArrayList(1);
-        String trackers = getOpenTrackerString();
-        StringTokenizer tok = new StringTokenizer(trackers, ", ");
-        while (tok.hasMoreTokens())
-            rv.add(tok.nextToken());
-        
-        if (rv.isEmpty())
-            return null;
-        return rv;
+            return Collections.EMPTY_LIST;
+        return _openTrackers;
+    }
+
+    /**
+     *  List of open trackers to use as backups even if disabled
+     *  @return non-null
+     *  @since 0.9.4
+     */
+    public List<String> getBackupTrackers() { 
+        return _openTrackers;
     }
     
     public void setUseOpenTrackers(boolean yes) {
@@ -474,6 +583,22 @@ public class I2PSnarkUtil {
     public boolean shouldUseOpenTrackers() {
         return _shouldUseOT;
     }
+    
+    /** @since DHT */
+    public synchronized void setUseDHT(boolean yes) {
+        _shouldUseDHT = yes;
+        if (yes && _manager != null && _dht == null) {
+            _dht = new KRPC(_context, _manager.getSession());
+        } else if (!yes && _dht != null) {
+            _dht.stop();
+            _dht = null;
+        }
+    }
+
+    /** @since DHT */
+    public boolean shouldUseDHT() {
+        return _shouldUseDHT;
+    }
 
     /**
      *  Like DataHelper.toHexString but ensures no loss of leading zero bytes
@@ -490,40 +615,6 @@ public class I2PSnarkUtil {
         return buf.toString();
     }
 
-    /** hook between snark's logger and an i2p log */
-    void debug(String msg, int snarkDebugLevel) {
-        debug(msg, snarkDebugLevel, null);
-    }
-    void debug(String msg, int snarkDebugLevel, Throwable t) {
-        if (t instanceof OutOfMemoryError) {
-            try { Thread.sleep(100); } catch (InterruptedException ie) {}
-            try {
-                t.printStackTrace();
-            } catch (Throwable tt) {}
-            try {
-                System.out.println("OOM thread: " + Thread.currentThread().getName());
-            } catch (Throwable tt) {}
-        }
-        switch (snarkDebugLevel) {
-            case 0:
-            case 1:
-                _log.error(msg, t);
-                break;
-            case 2:
-                _log.warn(msg, t);
-                break;
-            case 3:
-            case 4:
-                _log.info(msg, t);
-                break;
-            case 5:
-            case 6:
-            default:
-                _log.debug(msg, t);
-                break;
-        }
-    }
-
     private static final String BUNDLE_NAME = "org.klomp.snark.web.messages";
 
     /** lang in routerconsole.lang property, else current locale */

apps/i2psnark/java/src/org/klomp/snark/MagnetState.java

@@ -5,10 +5,10 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Random;
 
 import net.i2p.I2PAppContext;
 import net.i2p.data.DataHelper;
+import net.i2p.util.RandomSource;
 
 import org.klomp.snark.bencode.BDecoder;
 import org.klomp.snark.bencode.BEValue;
@@ -27,7 +27,6 @@ import org.klomp.snark.bencode.BEValue;
  */
 class MagnetState {
     public static final int CHUNK_SIZE = 16*1024;
-    private static final Random random = I2PAppContext.getGlobalContext().random();
 
     private final byte[] infohash;
     private boolean complete;
@@ -129,7 +128,7 @@ class MagnetState {
             throw new IllegalArgumentException("not initialized");
         if (complete)
             throw new IllegalArgumentException("complete");
-        int rand = random.nextInt(totalChunks);
+        int rand = RandomSource.getInstance().nextInt(totalChunks);
         for (int i = 0; i < totalChunks; i++) {
             int chk = (i + rand) % totalChunks; 
             if (!(have.get(chk) || requested.get(chk))) {

apps/i2psnark/java/src/org/klomp/snark/MagnetURI.java

@@ -0,0 +1,212 @@
+package org.klomp.snark;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+import net.i2p.data.Base32;
+
+/**
+ *
+ * @since 0.9.4 moved from I2PSnarkServlet
+ */
+public class MagnetURI {
+
+    private final String _tracker;
+    private final String _name;
+    private final byte[] _ih;
+
+    /** BEP 9 */
+    public static final String MAGNET = "magnet:";
+    public static final String MAGNET_FULL = MAGNET + "?xt=urn:btih:";
+    /** http://sponge.i2p/files/maggotspec.txt */
+    public static final String MAGGOT = "maggot://";
+
+    /**
+     *  @param url non-null
+     */
+    public MagnetURI(I2PSnarkUtil util, String url) throws IllegalArgumentException {
+        String ihash;
+        String name;
+        String trackerURL = null;
+        if (url.startsWith(MAGNET)) {
+            // magnet:?xt=urn:btih:0691e40aae02e552cfcb57af1dca56214680c0c5&tr=http://tracker2.postman.i2p/announce.php
+            String xt = getParam("xt", url);
+            if (xt == null || !xt.startsWith("urn:btih:"))
+                throw new IllegalArgumentException();
+            ihash = xt.substring("urn:btih:".length());
+            trackerURL = getTrackerParam(url);
+            name = util.getString("Magnet") + ' ' + ihash;
+            String dn = getParam("dn", url);
+            if (dn != null)
+                name += " (" + Storage.filterName(dn) + ')';
+        } else if (url.startsWith(MAGGOT)) {
+            // maggot://0691e40aae02e552cfcb57af1dca56214680c0c5:0b557bbdf8718e95d352fbe994dec3a383e2ede7
+            ihash = url.substring(MAGGOT.length()).trim();
+            int col = ihash.indexOf(':');
+            if (col >= 0)
+                ihash = ihash.substring(0, col);
+            name = util.getString("Magnet") + ' ' + ihash;
+        } else {
+            throw new IllegalArgumentException();
+        }
+        byte[] ih = null;
+        if (ihash.length() == 32) {
+            ih = Base32.decode(ihash);
+        } else if (ihash.length() == 40) {
+            //  Like DataHelper.fromHexString() but ensures no loss of leading zero bytes
+            ih = new byte[20];
+            try {
+                for (int i = 0; i < 20; i++) {
+                    ih[i] = (byte) (Integer.parseInt(ihash.substring(i*2, (i*2) + 2), 16) & 0xff);
+                }
+            } catch (NumberFormatException nfe) {
+                ih = null;
+            }
+        }
+        if (ih == null || ih.length != 20)
+            throw new IllegalArgumentException();
+        _ih = ih;
+        _name = name;
+        _tracker = trackerURL;
+    }
+
+    /**
+     *  @return 20 bytes or null
+     */
+    public byte[] getInfoHash() {
+        return _ih;
+    }
+
+    /**
+     *  @return pretty name or null
+     */
+    public String getName() {
+        return _name;
+    }
+
+    /**
+     *  @return tracker url or null
+     */
+    public String getTrackerURL() {
+        return _tracker;
+    }
+
+    /**
+     *  @return first decoded parameter or null
+     */
+    private static String getParam(String key, String uri) {
+        int idx = uri.indexOf('?' + key + '=');
+        if (idx >= 0) {
+            idx += key.length() + 2;
+        } else {
+            idx = uri.indexOf('&' + key + '=');
+            if (idx >= 0)
+                idx += key.length() + 2;
+        }
+        if (idx < 0 || idx > uri.length())
+            return null;
+        String rv = uri.substring(idx);
+        idx = rv.indexOf('&');
+        if (idx >= 0)
+            rv = rv.substring(0, idx);
+        else
+            rv = rv.trim();
+        return decode(rv);
+    }
+
+    /**
+     *  @return all decoded parameters or null
+     *  @since 0.9.1
+     */
+    private static List<String> getMultiParam(String key, String uri) {
+        int idx = uri.indexOf('?' + key + '=');
+        if (idx >= 0) {
+            idx += key.length() + 2;
+        } else {
+            idx = uri.indexOf('&' + key + '=');
+            if (idx >= 0)
+                idx += key.length() + 2;
+        }
+        if (idx < 0 || idx > uri.length())
+            return null;
+        List<String> rv = new ArrayList();
+        while (true) {
+            String p = uri.substring(idx);
+            uri = p;
+            idx = p.indexOf('&');
+            if (idx >= 0)
+                p = p.substring(0, idx);
+            else
+                p = p.trim();
+            rv.add(decode(p));
+            idx = uri.indexOf('&' + key + '=');
+            if (idx < 0)
+                break;
+            idx += key.length() + 2;
+        }
+        return rv;
+    }
+
+    /**
+     *  @return first valid I2P tracker or null
+     *  @since 0.9.1
+     */
+    private static String getTrackerParam(String uri) {
+        List<String> trackers = getMultiParam("tr", uri);
+        if (trackers == null)
+            return null;
+        for (String t : trackers) {
+            try {
+                URI u = new URI(t);
+                String protocol = u.getScheme();
+                String host = u.getHost();
+                if (protocol == null || host == null ||
+                    !protocol.toLowerCase(Locale.US).equals("http") ||
+                    !host.toLowerCase(Locale.US).endsWith(".i2p"))
+                    continue;
+                return t;
+            } catch(URISyntaxException use) {}
+        }
+        return null;
+    }
+
+    /**
+     *  Decode %xx encoding, convert to UTF-8 if necessary
+     *  Copied from i2ptunnel LocalHTTPServer
+     *  @since 0.9.1
+     */
+    private static String decode(String s) {
+        if (!s.contains("%"))
+            return s;
+        StringBuilder buf = new StringBuilder(s.length());
+        boolean utf8 = false;
+        for (int i = 0; i < s.length(); i++) {
+            char c = s.charAt(i);
+            if (c != '%') {
+                buf.append(c);
+            } else {
+                try {
+                    int val = Integer.parseInt(s.substring(++i, (++i) + 1), 16);
+                    if ((val & 0x80) != 0)
+                        utf8 = true;
+                    buf.append((char) val);
+                } catch (IndexOutOfBoundsException ioobe) {
+                    break;
+                } catch (NumberFormatException nfe) {
+                    break;
+                }
+            }
+        }
+        if (utf8) {
+            try {
+                return new String(buf.toString().getBytes("ISO-8859-1"), "UTF-8");
+            } catch (UnsupportedEncodingException uee) {}
+        }
+        return buf.toString();
+    }
+
+}

apps/i2psnark/java/src/org/klomp/snark/Message.java

@@ -23,10 +23,13 @@ package org.klomp.snark;
 import java.io.DataOutputStream;
 import java.io.IOException;
 
-import net.i2p.util.SimpleTimer;
+import net.i2p.data.ByteArray;
+import net.i2p.util.ByteCache;
 
-// Used to queue outgoing connections
-// sendMessage() should be used to translate them to wire format.
+/**
+ * Used to queue outgoing connections
+ * sendMessage() should be used to translate them to wire format.
+ */
 class Message
 {
   final static byte KEEP_ALIVE   = -1;
@@ -71,6 +74,9 @@ class Message
   // now unused
   //SimpleTimer.TimedEvent expireEvent;
   
+  private static final int BUFSIZE = PeerState.PARTSIZE;
+  private static final ByteCache _cache = ByteCache.getInstance(16, BUFSIZE);
+
   /** Utility method for sending a message through a DataStream. */
   void sendMessage(DataOutputStream dos) throws IOException
   {
@@ -81,11 +87,15 @@ class Message
         return;
       }
 
+    ByteArray ba;
     // Get deferred data
     if (data == null && dataLoader != null) {
-        data = dataLoader.loadData(piece, begin, length);
-        if (data == null)
+        ba = dataLoader.loadData(piece, begin, length);
+        if (ba == null)
             return;  // hmm will get retried, but shouldn't happen
+        data = ba.getData();
+    } else {
+        ba = null;
     }
 
     // Calculate the total length in bytes
@@ -141,6 +151,10 @@ class Message
     // Send actual data
     if (type == BITFIELD || type == PIECE || type == EXTENSION)
       dos.write(data, off, len);
+
+    // Was pulled from cache in Storage.getPiece() via dataLoader
+    if (ba != null && ba.getData().length == BUFSIZE)
+        _cache.release(ba, false);
   }
 
     @Override

apps/i2psnark/java/src/org/klomp/snark/MetaInfo.java

@@ -33,7 +33,6 @@ import java.util.Map;
 
 import net.i2p.I2PAppContext;
 import net.i2p.crypto.SHA1;
-import net.i2p.data.Base64;
 import net.i2p.data.DataHelper;
 import net.i2p.util.Log;
 
@@ -61,6 +60,8 @@ public class MetaInfo
   private final int piece_length;
   private final byte[] piece_hashes;
   private final long length;
+  private final boolean privateTorrent;
+  private final List<List<String>> announce_list;
   private Map<String, BEValue> infoMap;
 
   /**
@@ -69,9 +70,11 @@ public class MetaInfo
    *  @param announce may be null
    *  @param files null for single-file torrent
    *  @param lengths null for single-file torrent
+   *  @param announce_list may be null
    */
   MetaInfo(String announce, String name, String name_utf8, List<List<String>> files, List<Long> lengths,
-           int piece_length, byte[] piece_hashes, long length)
+           int piece_length, byte[] piece_hashes, long length, boolean privateTorrent,
+           List<List<String>> announce_list)
   {
     this.announce = announce;
     this.name = name;
@@ -82,6 +85,8 @@ public class MetaInfo
     this.piece_length = piece_length;
     this.piece_hashes = piece_hashes;
     this.length = length;
+    this.privateTorrent = privateTorrent;
+    this.announce_list = announce_list;
 
     // TODO if we add a parameter for other keys
     //if (other != null) {
@@ -140,6 +145,23 @@ public class MetaInfo
         this.announce = val.getString();
     }
 
+    // BEP 12
+    val = m.get("announce-list");
+    if (val == null) {
+        this.announce_list = null;
+    } else {
+        this.announce_list = new ArrayList();
+        List<BEValue> bl1 = val.getList();
+        for (BEValue bev : bl1) {
+            List<BEValue> bl2 = bev.getList();
+            List<String> sl2 = new ArrayList();           
+            for (BEValue bev2 : bl2) {
+                sl2.add(bev2.getString());
+            }
+            this.announce_list.add(sl2);
+        }
+    }
+
     val = m.get("info");
     if (val == null)
         throw new InvalidBEncodingException("Missing info map");
@@ -160,6 +182,10 @@ public class MetaInfo
     else
         name_utf8 = null;
 
+    // BEP 27
+    val = info.get("private");
+    privateTorrent = val != null && val.getString().equals("1");
+
     val = info.get("piece length");
     if (val == null)
         throw new InvalidBEncodingException("Missing piece length number");
@@ -212,7 +238,7 @@ public class MetaInfo
             if (l < oldTotal)
               throw new InvalidBEncodingException("Huge total length");
 
-            val = (BEValue)desc.get("path");
+            val = desc.get("path");
             if (val == null)
               throw new InvalidBEncodingException("Missing path list");
             List<BEValue> path_list = val.getList();
@@ -238,7 +264,7 @@ public class MetaInfo
 
             m_files.add(Collections.unmodifiableList(file));
             
-            val = (BEValue)desc.get("path.utf-8");
+            val = desc.get("path.utf-8");
             if (val != null) {
                 path_list = val.getList();
                 path_length = path_list.size();
@@ -291,6 +317,15 @@ public class MetaInfo
     return announce;
   }
 
+  /**
+   * Returns a list of lists of urls.
+   *
+   * @since 0.9.5
+   */
+  public List<List<String>> getAnnounceList() {
+    return announce_list;
+  }
+
   /**
    * Returns the original 20 byte SHA1 hash over the bencoded info map.
    */
@@ -318,6 +353,14 @@ public class MetaInfo
     return name;
   }
 
+  /**
+   * Is it a private torrent?
+   * @since 0.9
+   */
+  public boolean isPrivate() {
+    return privateTorrent;
+  }
+
   /**
    * Returns a list of lists of file name hierarchies or null if it is
    * a single name. It has the same size as the list returned by
@@ -409,6 +452,29 @@ public class MetaInfo
         return false;
     return true;
   }
+  
+  /**
+   *  @return good
+   *  @since 0.9.1
+   */
+  boolean checkPiece(PartialPiece pp) {
+    MessageDigest sha1 = SHA1.getInstance();
+    int piece = pp.getPiece();
+    byte[] hash;
+    try {
+        hash = pp.getHash();
+    } catch (IOException ioe) {
+        // Could be caused by closing a peer connnection
+        // we don't want the exception to propagate through
+        // to Storage.putPiece()
+        _log.warn("Error checking", ioe);
+        return false;
+    }
+    for (int i = 0; i < 20; i++)
+      if (hash[i] != piece_hashes[20 * piece + i])
+        return false;
+    return true;
+  }
 
   /**
    * Returns the total length of the torrent in bytes.
@@ -434,12 +500,13 @@ public class MetaInfo
   /**
    * Creates a copy of this MetaInfo that shares everything except the
    * announce URL.
+   * Drops any announce-list.
    */
   public MetaInfo reannounce(String announce)
   {
     return new MetaInfo(announce, name, name_utf8, files,
                         lengths, piece_length,
-                        piece_hashes, length);
+                        piece_hashes, length, privateTorrent, null);
   }
 
   /**
@@ -450,6 +517,8 @@ public class MetaInfo
         Map m = new HashMap();
         if (announce != null)
             m.put("announce", announce);
+        if (announce_list != null)
+            m.put("announce-list", announce_list);
         Map info = createInfoMap();
         m.put("info", info);
         // don't save this locally, we should only do this once
@@ -475,6 +544,10 @@ public class MetaInfo
     info.put("name", name);
     if (name_utf8 != null)
         info.put("name.utf-8", name_utf8);
+    // BEP 27
+    if (privateTorrent)
+        info.put("private", "1");
+
     info.put("piece length", Integer.valueOf(piece_length));
     info.put("pieces", piece_hashes);
     if (files == null)

apps/i2psnark/java/src/org/klomp/snark/PartialPiece.java

@@ -1,6 +1,24 @@
 package org.klomp.snark;
 
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.security.MessageDigest;
+
+import net.i2p.I2PAppContext;
+import net.i2p.crypto.SHA1;
+import net.i2p.data.ByteArray;
+import net.i2p.util.ByteCache;
+import net.i2p.util.Log;
+import net.i2p.util.SecureFile;
+
 /**
+ * Store the received data either on the heap or in a temp file.
+ * The third option, to write chunks directly to the destination file,
+ * is unimplemented.
+ *
  * This is the class passed from PeerCoordinator to PeerState so
  * PeerState may start requests.
  *
@@ -8,45 +26,84 @@ package org.klomp.snark;
  * a piece is not completely downloaded, for example
  * when the Peer disconnects or chokes.
  *
+ * New objects for the same piece are created during the end game -
+ * this object should not be shared among multiple peers.
+ *
  * @since 0.8.2
  */
 class PartialPiece implements Comparable {
 
-    private final int piece;
+    // we store the piece so we can use it in compareTo()
+    private final Piece piece;
+    // null if using temp file
     private final byte[] bs;
-    private final int off;
-    private final long createdTime;
+    private int off;
+    //private final long createdTime;
+    private File tempfile;
+    private RandomAccessFile raf;
+    private final int pclen;
+    private final File tempDir;
+
+    private static final int BUFSIZE = PeerState.PARTSIZE;
+    private static final ByteCache _cache = ByteCache.getInstance(16, BUFSIZE);
+
+    // Any bigger than this, use temp file instead of heap
+    private static final int MAX_IN_MEM = 128 * 1024;
+    // May be reduced on OOM
+    private static int _max_in_mem = MAX_IN_MEM;
 
     /**
      * Used by PeerCoordinator.
      * Creates a new PartialPiece, with no chunks yet downloaded.
-     * Allocates the data.
+     * Allocates the data storage area, either on the heap or in the
+     * temp directory, depending on size.
      *
      * @param piece Piece number requested.
      * @param len must be equal to the piece length
      */
-    public PartialPiece (int piece, int len) throws OutOfMemoryError {
+    public PartialPiece (Piece piece, int len, File tempDir) {
         this.piece = piece;
-        this.bs = new byte[len];
-        this.off = 0;
-        this.createdTime = 0;
+        this.pclen = len;
+        //this.createdTime = 0;
+        this.tempDir = tempDir;
+
+        // temps for finals
+        byte[] tbs = null;
+        try {
+            if (len <= MAX_IN_MEM) {
+                try {
+                    tbs = new byte[len];
+                    return;
+                } catch (OutOfMemoryError oom) {
+                    if (_max_in_mem > PeerState.PARTSIZE)
+                        _max_in_mem /= 2;
+                    Log log = I2PAppContext.getGlobalContext().logManager().getLog(PartialPiece.class);
+                    log.logAlways(Log.WARN, "OOM creating new partial piece");
+                    // fall through to use temp file
+                }
+            }
+            // delay creating temp file until required in read()
+        } finally {
+            // finals
+            this.bs = tbs;
+        }
     }
 
     /**
-     * Used by PeerState.
-     * Creates a new PartialPiece, with chunks up to but not including
-     * firstOutstandingRequest already downloaded and stored in the Request byte array.
+     *  Caller must synchronize
      *
-     * Note that this cannot handle gaps; chunks after a missing chunk cannot be saved.
-     * That would be harder.
-     *
-     * @param firstOutstandingRequest the first request not fulfilled for the piece
+     *  @since 0.9.1
      */
-    public PartialPiece (Request firstOutstandingRequest) {
-        this.piece = firstOutstandingRequest.piece;
-        this.bs = firstOutstandingRequest.bs;
-        this.off = firstOutstandingRequest.off;
-        this.createdTime = System.currentTimeMillis();
+    private void createTemp() throws IOException {
+        //tfile = SecureFile.createTempFile("piece", null, tempDir);
+        // debug
+        tempfile = SecureFile.createTempFile("piece_" + piece.getId() + '_', null, tempDir);
+        //I2PAppContext.getGlobalContext().logManager().getLog(PartialPiece.class).warn("Created " + tempfile);
+        // tfile.deleteOnExit() ???
+        raf = new RandomAccessFile(tempfile, "rw");
+        // Do not preallocate the file space.
+        // Not necessary to call setLength(), file is extended when written
+        //traf.setLength(len);
     }
 
     /**
@@ -55,33 +112,200 @@ class PartialPiece implements Comparable {
      */
 
     public Request getRequest() {
-        return new Request(this.piece, this.bs, this.off, Math.min(this.bs.length - this.off, PeerState.PARTSIZE));
+        return new Request(this, this.off, Math.min(this.pclen - this.off, PeerState.PARTSIZE));
     }
 
     /** piece number */
     public int getPiece() {
-         return this.piece;
+         return this.piece.getId();
     }
 
-    /** how many bytes are good */
+    /**
+     *  @since 0.9.1
+     */
+    public int getLength() {
+         return this.pclen;
+    }
+
+    /**
+     *  How many bytes are good - only valid by setDownloaded()
+     */
     public int getDownloaded() {
          return this.off;
     }
 
+    /**
+     *  Call this before returning a PartialPiece to the PeerCoordinator
+     *  @since 0.9.1
+     */
+    public void setDownloaded(int offset) {
+         this.off = offset;
+    }
+
+/****
     public long getCreated() {
          return this.createdTime;
     }
+****/
 
     /**
-     *  Highest downloaded first
+     *  Piece must be complete.
+     *  The SHA1 hash of the completely read data.
+     *  @since 0.9.1
+     */
+    public byte[] getHash() throws IOException {
+        MessageDigest sha1 = SHA1.getInstance();
+        if (bs != null) {
+            sha1.update(bs);
+        } else {
+            int read = 0;
+            int buflen = Math.min(pclen, BUFSIZE);
+            ByteArray ba;
+            byte[] buf;
+            if (buflen == BUFSIZE) {
+                ba = _cache.acquire();
+                buf = ba.getData();
+            } else {
+                ba = null;
+                buf = new byte[buflen];
+            }
+            synchronized (this) {
+                if (raf == null)
+                    throw new IOException();
+                raf.seek(0);
+                while (read < pclen) {
+                    int rd = raf.read(buf, 0, Math.min(buf.length, pclen - read));
+                    if (rd < 0)
+                        break;
+                    read += rd;
+                    sha1.update(buf, 0, rd);
+                }
+            }
+            if (ba != null)
+                _cache.release(ba, false);
+            if (read < pclen)
+                throw new IOException();
+        }
+        return sha1.digest();
+    }
+    
+    /**
+     *  Blocking.
+     *  @since 0.9.1
+     */
+    public void read(DataInputStream din, int off, int len) throws IOException {
+        if (bs != null) {
+            din.readFully(bs, off, len);
+        } else {
+            // read in fully before synching on raf
+            ByteArray ba;
+            byte[] tmp;
+            if (len == BUFSIZE) {
+                ba = _cache.acquire();
+                tmp = ba.getData();
+            } else {
+                ba = null;
+                tmp = new byte[len];
+            }
+            din.readFully(tmp);
+            synchronized (this) {
+                if (raf == null)
+                    createTemp();
+                raf.seek(off);
+                raf.write(tmp);
+            }
+            if (ba != null)
+                _cache.release(ba, false);
+        }
+    }
+
+    /**
+     *  Piece must be complete.
+     *  Caller must synchronize on out and seek to starting point.
+     *  Caller must call release() when done with the whole piece.
+     *
+     *  @param out stream to write to
+     *  @param offset offset in the piece
+     *  @param len length to write
+     *  @since 0.9.1
+     */
+    public void write(DataOutput out, int offset, int len) throws IOException {
+        if (bs != null) {
+            out.write(bs, offset, len);
+        } else {
+            int read = 0;
+            int buflen = Math.min(len, BUFSIZE);
+            ByteArray ba;
+            byte[] buf;
+            if (buflen == BUFSIZE) {
+                ba = _cache.acquire();
+                buf = ba.getData();
+            } else {
+                ba = null;
+                buf = new byte[buflen];
+            }
+            synchronized (this) {
+                if (raf == null)
+                    throw new IOException();
+                raf.seek(offset);
+                while (read < len) {
+                    int rd = Math.min(buf.length, len - read);
+                    raf.readFully(buf, 0, rd);
+                    read += rd;
+                    out.write(buf, 0, rd);
+                }
+            }
+            if (ba != null)
+                _cache.release(ba, false);
+        }
+    }
+    
+    /**
+     *  Release all resources.
+     *
+     *  @since 0.9.1
+     */
+    public void release() {
+        if (bs == null) {
+            synchronized (this) {
+                if (raf != null)
+                    locked_release();
+            }
+            //if (raf != null)
+            //    I2PAppContext.getGlobalContext().logManager().getLog(PartialPiece.class).warn("Released " + tempfile);
+        }
+    }
+    
+    /**
+     *  Caller must synchronize
+     *
+     *  @since 0.9.1
+     */
+    private void locked_release() {
+        try {
+            raf.close();
+        } catch (IOException ioe) {
+            I2PAppContext.getGlobalContext().logManager().getLog(PartialPiece.class).warn("Error closing " + raf, ioe);
+        }
+        tempfile.delete();
+    }
+
+    /*
+     *  Highest priority first,
+     *  then rarest first,
+     *  then highest downloaded first
      */
     public int compareTo(Object o) throws ClassCastException {
-        return ((PartialPiece)o).off - this.off;  // reverse
+        PartialPiece opp = (PartialPiece)o;
+        int d = this.piece.compareTo(opp.piece);
+        if (d != 0)
+            return d;
+        return opp.off - this.off;  // reverse
     }
     
     @Override
     public int hashCode() {
-        return piece * 7777;
+        return piece.getId() * 7777;
     }
 
     /**
@@ -92,13 +316,13 @@ class PartialPiece implements Comparable {
     public boolean equals(Object o) {
         if (o instanceof PartialPiece) {
             PartialPiece pp = (PartialPiece)o;
-            return pp.piece == this.piece;
+            return pp.piece.getId() == this.piece.getId();
         }
         return false;
     }
 
     @Override
     public String toString() {
-        return "Partial(" + piece + ',' + off + ',' + bs.length + ')';
+        return "Partial(" + piece.getId() + ',' + off + ',' + pclen + ')';
     }
 }

apps/i2psnark/java/src/org/klomp/snark/Peer.java

@@ -80,7 +80,9 @@ public class Peer implements Comparable
   static final long OPTION_FAST      = 0x0000000000000004l;
   static final long OPTION_DHT       = 0x0000000000000001l;
   /** we use a different bit since the compact format is different */
+/* no, let's use an extension message
   static final long OPTION_I2P_DHT   = 0x0000000040000000l;
+*/
   static final long OPTION_AZMP      = 0x1000000000000000l;
   private long options;
 
@@ -268,15 +270,18 @@ public class Peer implements Comparable
             if (_log.shouldLog(Log.DEBUG))
                 _log.debug("Peer supports extensions, sending reply message");
             int metasize = metainfo != null ? metainfo.getInfoBytes().length : -1;
-            out.sendExtension(0, ExtensionHandler.getHandshake(metasize));
+            boolean pexAndMetadata = metainfo == null || !metainfo.isPrivate();
+            boolean dht = util.getDHT() != null;
+            out.sendExtension(0, ExtensionHandler.getHandshake(metasize, pexAndMetadata, dht));
         }
 
-        if ((options & OPTION_I2P_DHT) != 0 && util.getDHT() != null) {
-            if (_log.shouldLog(Log.DEBUG))
-                _log.debug("Peer supports DHT, sending PORT message");
-            int port = util.getDHT().getPort();
-            out.sendPort(port);
-        }
+        // Old DHT PORT message
+        //if ((options & OPTION_I2P_DHT) != 0 && util.getDHT() != null) {
+        //    if (_log.shouldLog(Log.DEBUG))
+        //        _log.debug("Peer supports DHT, sending PORT message");
+        //    int port = util.getDHT().getPort();
+        //    out.sendPort(port);
+        //}
 
         // Send our bitmap
         if (bitfield != null)
@@ -459,7 +464,7 @@ public class Peer implements Comparable
         if (this.deregister) {
           PeerListener p = s.listener;
           if (p != null) {
-            List<PartialPiece> pcs = s.returnPartialPieces();
+            List<Request> pcs = s.returnPartialPieces();
             if (!pcs.isEmpty())
                 p.savePartialPieces(this, pcs);
             // now covered by savePartialPieces

apps/i2psnark/java/src/org/klomp/snark/PeerAcceptor.java

@@ -46,6 +46,10 @@ public class PeerAcceptor
   private final PeerCoordinator coordinator;
   final PeerCoordinatorSet coordinators;
 
+  /** shorten timeout while reading handshake */
+  private static final long HASH_READ_TIMEOUT = 45*1000;
+
+
   public PeerAcceptor(PeerCoordinator coordinator)
   {
     this.coordinator = coordinator;
@@ -69,11 +73,20 @@ public class PeerAcceptor
     // talk about, and we can just look for that in our list of active torrents.
     byte peerInfoHash[] = null;
     if (in instanceof BufferedInputStream) {
+        // multitorrent
         in.mark(LOOKAHEAD_SIZE);
-        peerInfoHash = readHash(in);
+        long timeout = socket.getReadTimeout();
+        socket.setReadTimeout(HASH_READ_TIMEOUT);
+        try {
+            peerInfoHash = readHash(in);
+        } catch (IOException ioe) {
+            // unique exception so ConnectionAcceptor can blame the peer
+            throw new ProtocolException(ioe.toString());
+        }
+        socket.setReadTimeout(timeout);
         in.reset();
     } else {
-        // is this working right?
+        // Single torrent - is this working right?
         try {
           peerInfoHash = readHash(in);
           if (_log.shouldLog(Log.INFO))
@@ -104,9 +117,8 @@ public class PeerAcceptor
         }
     } else {
         // multitorrent capable, so lets see what we can handle
-        for (Iterator iter = coordinators.iterator(); iter.hasNext(); ) {
-            PeerCoordinator cur = (PeerCoordinator)iter.next();
-
+        PeerCoordinator cur = coordinators.get(peerInfoHash);
+        if (cur != null) {
             if (DataHelper.eq(cur.getInfoHash(), peerInfoHash)) {
                 if (cur.needPeers())
                   {
@@ -130,22 +142,50 @@ public class PeerAcceptor
     }
   }
 
+  private static final String PROTO_STR = "BitTorrent protocol";
+  private static final int PROTO_STR_LEN = PROTO_STR.length();
+  private static final int PROTO_LEN = PROTO_STR_LEN + 1;
+  private static final int[] PROTO = new int[PROTO_LEN];
+  static {
+      PROTO[0] = PROTO_STR_LEN;
+      for (int i = 0; i < PROTO_STR_LEN; i++) {
+          PROTO[i+1] = PROTO_STR.charAt(i);
+      }
+  }
+
   /** 48 */
-  private static final int LOOKAHEAD_SIZE = 1 + // chr(19)
-                                            "BitTorrent protocol".length() +
+  private static final int LOOKAHEAD_SIZE = PROTO_LEN +
                                             8 + // blank, reserved
                                             20; // infohash
 
   /** 
-   * Read ahead to the infohash, throwing an exception if there isn't enough data
+   * Read ahead to the infohash, throwing an exception if there isn't enough data.
+   * Also check the first 20 bytes for the correct protocol here and throw IOE if bad,
+   * so we don't hang waiting for 48 bytes if it's not a bittorrent client.
+   * The 20 bytes are checked again in Peer.handshake().
    */
-  private byte[] readHash(InputStream in) throws IOException {
-    byte buf[] = new byte[LOOKAHEAD_SIZE];
+  private static byte[] readHash(InputStream in) throws IOException {
+    for (int i = 0; i < PROTO_LEN; i++) {
+        int b = in.read();
+        if (b != PROTO[i])
+            throw new IOException("Bad protocol 0x" + Integer.toHexString(b) + " at byte " + i);
+    }
+    if (in.skip(8) != 8)
+        throw new IOException("EOF before hash");
+    byte buf[] = new byte[20];
     int read = DataHelper.read(in, buf);
     if (read != buf.length)
         throw new IOException("Unable to read the hash (read " + read + ")");
-    byte rv[] = new byte[20];
-    System.arraycopy(buf, buf.length-rv.length, rv, 0, rv.length);
-    return rv;
+    return buf;
   }
+
+    /** 
+     *  A unique exception so we can tell the ConnectionAcceptor about non-BT connections
+     *  @since 0.9.1
+     */
+    public static class ProtocolException extends IOException {
+        public ProtocolException(String s) {
+            super(s);
+        }
+    }
 }

apps/i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java

@@ -21,11 +21,14 @@
 package org.klomp.snark;
 
 import java.util.ArrayList;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Random;
 
 import net.i2p.I2PAppContext;
+import net.i2p.data.DataHelper;
+import net.i2p.util.Log;
+
+import org.klomp.snark.dht.DHT;
 
 /**
  * TimerTask that checks for good/bad up/downloader. Works together
@@ -37,16 +40,18 @@ class PeerCheckerTask implements Runnable
 
   private final PeerCoordinator coordinator;
   private final I2PSnarkUtil _util;
+  private final Log _log;
+  private final Random random;
   private int _runCount;
 
   PeerCheckerTask(I2PSnarkUtil util, PeerCoordinator coordinator)
   {
     _util = util;
+    _log = util.getContext().logManager().getLog(PeerCheckerTask.class);
+    random = util.getContext().random();
     this.coordinator = coordinator;
   }
 
-  private static final Random random = I2PAppContext.getGlobalContext().random();
-
   public void run()
   {
         _runCount++;
@@ -60,9 +65,7 @@ class PeerCheckerTask implements Runnable
         long worstdownload = Long.MAX_VALUE;
         Peer worstDownloader = null;
 
-        int peers = 0;
         int uploaders = 0;
-        int downloaders = 0;
         int removedCount = 0;
 
         long uploaded = 0;
@@ -73,6 +76,7 @@ class PeerCheckerTask implements Runnable
         List<Peer> removed = new ArrayList();
         int uploadLimit = coordinator.allowedUploaders();
         boolean overBWLimit = coordinator.overUpBWLimit();
+        DHT dht = _util.getDHT();
         for (Peer peer : peerList) {
 
             // Remove dying peers
@@ -85,12 +89,16 @@ class PeerCheckerTask implements Runnable
                 continue;
               }
 
-            peers++;
+            if (peer.getInactiveTime() > PeerCoordinator.MAX_INACTIVE) {
+                if (_log.shouldLog(Log.WARN))
+                    _log.warn("Disconnecting peer idle " +
+                              DataHelper.formatDuration(peer.getInactiveTime()) + ": " + peer);
+                peer.disconnect();
+                continue;
+            }
 
             if (!peer.isChoking())
               uploaders++;
-            if (!peer.isChoked() && peer.isInteresting())
-              downloaders++;
 
             long upload = peer.getUploaded();
             uploaded += upload;
@@ -99,14 +107,15 @@ class PeerCheckerTask implements Runnable
 	    peer.setRateHistory(upload, download);
             peer.resetCounters();
 
-            _util.debug(peer + ":", Snark.DEBUG);
-            _util.debug(" ul: " + upload*1024/KILOPERSECOND
+            if (_log.shouldLog(Log.DEBUG)) {
+                _log.debug(peer + ":"
+                        + " ul: " + upload*1024/KILOPERSECOND
                         + " dl: " + download*1024/KILOPERSECOND
                         + " i: " + peer.isInterested()
                         + " I: " + peer.isInteresting()
                         + " c: " + peer.isChoking()
-                        + " C: " + peer.isChoked(),
-                        Snark.DEBUG);
+                        + " C: " + peer.isChoked());
+            }
 
             // Choke a percentage of them rather than all so it isn't so drastic...
             // unless this torrent is over the limit all by itself.
@@ -127,8 +136,8 @@ class PeerCheckerTask implements Runnable
                 // Check if it still wants pieces from us.
                 if (!peer.isInterested())
                   {
-                    _util.debug("Choke uninterested peer: " + peer,
-                                Snark.INFO);
+                    if (_log.shouldLog(Log.DEBUG))
+                        _log.debug("Choke uninterested peer: " + peer);
                     peer.setChoking(true);
                     uploaders--;
                     coordinator.uploaders--;
@@ -138,8 +147,8 @@ class PeerCheckerTask implements Runnable
                   }
                 else if (overBWLimitChoke)
                   {
-                    _util.debug("BW limit (" + upload + "/" + uploaded + "), choke peer: " + peer,
-                                Snark.INFO);
+                    if (_log.shouldLog(Log.DEBUG))
+                        _log.debug("BW limit (" + upload + "/" + uploaded + "), choke peer: " + peer);
                     peer.setChoking(true);
                     uploaders--;
                     coordinator.uploaders--;
@@ -151,7 +160,8 @@ class PeerCheckerTask implements Runnable
                 else if (peer.isInteresting() && peer.isChoked())
                   {
                     // If they are choking us make someone else a downloader
-                    _util.debug("Choke choking peer: " + peer, Snark.DEBUG);
+                    if (_log.shouldLog(Log.DEBUG))
+                        _log.debug("Choke choking peer: " + peer);
                     peer.setChoking(true);
                     uploaders--;
                     coordinator.uploaders--;
@@ -163,7 +173,8 @@ class PeerCheckerTask implements Runnable
                 else if (!peer.isInteresting() && !coordinator.completed())
                   {
                     // If they aren't interesting make someone else a downloader
-                    _util.debug("Choke uninteresting peer: " + peer, Snark.DEBUG);
+                    if (_log.shouldLog(Log.DEBUG))
+                        _log.debug("Choke uninteresting peer: " + peer);
                     peer.setChoking(true);
                     uploaders--;
                     coordinator.uploaders--;
@@ -177,8 +188,8 @@ class PeerCheckerTask implements Runnable
                          && download == 0)
                   {
                     // We are downloading but didn't receive anything...
-                    _util.debug("Choke downloader that doesn't deliver:"
-                                + peer, Snark.DEBUG);
+                    if (_log.shouldLog(Log.DEBUG))
+                        _log.debug("Choke downloader that doesn't deliver: " + peer);
                     peer.setChoking(true);
                     uploaders--;
                     coordinator.uploaders--;
@@ -205,10 +216,13 @@ class PeerCheckerTask implements Runnable
             // send PEX
             if ((_runCount % 17) == 0 && !peer.isCompleted())
                 coordinator.sendPeers(peer);
-            peer.keepAlive();
+            // cheap failsafe for seeds connected to seeds, stop pinging and hopefully
+            // the inactive checker (above) will eventually disconnect it
+            if (coordinator.getNeededLength() > 0 || !peer.isCompleted())
+                peer.keepAlive();
             // announce them to local tracker (TrackerClient does this too)
-            if (_util.getDHT() != null && (_runCount % 5) == 0) {
-                _util.getDHT().announce(coordinator.getInfoHash(), peer.getPeerID().getDestHash());
+            if (dht != null && (_runCount % 5) == 0) {
+                dht.announce(coordinator.getInfoHash(), peer.getPeerID().getDestHash());
             }
           }
 
@@ -222,8 +236,8 @@ class PeerCheckerTask implements Runnable
             || uploaders > uploadLimit)
             && worstDownloader != null)
           {
-            _util.debug("Choke worst downloader: " + worstDownloader,
-                        Snark.DEBUG);
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("Choke worst downloader: " + worstDownloader);
 
             worstDownloader.setChoking(true);
             coordinator.uploaders--;
@@ -256,8 +270,8 @@ class PeerCheckerTask implements Runnable
         }
 
         // announce ourselves to local tracker (TrackerClient does this too)
-        if (_util.getDHT() != null && (_runCount % 16) == 0) {
-            _util.getDHT().announce(coordinator.getInfoHash());
+        if (dht != null && (_runCount % 16) == 0) {
+            dht.announce(coordinator.getInfoHash());
         }
   }
 }

apps/i2psnark/java/src/org/klomp/snark/PeerConnectionIn.java

@@ -148,11 +148,9 @@ class PeerConnectionIn implements Runnable
                 begin = din.readInt();
                 len = i-9;
                 Request req = ps.getOutstandingRequest(piece, begin, len);
-                byte[] piece_bytes;
                 if (req != null)
                   {
-                    piece_bytes = req.bs;
-                    din.readFully(piece_bytes, begin, len);
+                    req.read(din);
                     ps.pieceMessage(req);
                     if (_log.shouldLog(Log.DEBUG)) 
                         _log.debug("Received data(" + piece + "," + begin + ") from " + peer);
@@ -160,8 +158,9 @@ class PeerConnectionIn implements Runnable
                 else
                   {
                     // XXX - Consume but throw away afterwards.
-                    piece_bytes = new byte[len];
-                    din.readFully(piece_bytes);
+                    int rcvd = din.skipBytes(len);
+                    if (rcvd != len)
+                        throw new IOException("EOF reading unwanted data");
                     if (_log.shouldLog(Log.DEBUG)) 
                         _log.debug("Received UNWANTED data(" + piece + "," + begin + ") from " + peer);
                   }

apps/i2psnark/java/src/org/klomp/snark/PeerConnectionOut.java

@@ -143,7 +143,7 @@ class PeerConnectionOut implements Runnable
                           }
                       }
                     if (m == null && !sendQueue.isEmpty()) {
-                      m = (Message)sendQueue.remove(0);
+                      m = sendQueue.remove(0);
                       //SimpleTimer.getInstance().removeEvent(m.expireEvent);
                     }
                   }
@@ -395,7 +395,7 @@ class PeerConnectionOut implements Runnable
         while (it.hasNext())
           {
             Message m = (Message)it.next();
-            if (m.type == Message.REQUEST && m.piece == req.piece &&
+            if (m.type == Message.REQUEST && m.piece == req.getPiece() &&
                 m.begin == req.off && m.length == req.len)
               {
                 if (_log.shouldLog(Log.DEBUG))
@@ -406,7 +406,7 @@ class PeerConnectionOut implements Runnable
       }
     Message m = new Message();
     m.type = Message.REQUEST;
-    m.piece = req.piece;
+    m.piece = req.getPiece();
     m.begin = req.off;
     m.length = req.len;
     addMessage(m);
@@ -437,6 +437,7 @@ class PeerConnectionOut implements Runnable
    */
   void sendPiece(int piece, int begin, int length, DataLoader loader)
   {
+    /****
       boolean sendNow = false;
       // are there any cases where we should?
 
@@ -447,6 +448,7 @@ class PeerConnectionOut implements Runnable
             sendPiece(piece, begin, length, bytes);
         return;
       }
+    ****/
 
       // queue a fake message... set everything up,
       // except save the PeerState instead of the bytes.
@@ -492,7 +494,7 @@ class PeerConnectionOut implements Runnable
           {
             Message m = (Message)it.next();
             if (m.type == Message.REQUEST
-                && m.piece == req.piece
+                && m.piece == req.getPiece()
                 && m.begin == req.off
                 && m.length == req.len)
               it.remove();
@@ -502,7 +504,7 @@ class PeerConnectionOut implements Runnable
     // Always send, just to be sure it it is really canceled.
     Message m = new Message();
     m.type = Message.CANCEL;
-    m.piece = req.piece;
+    m.piece = req.getPiece();
     m.begin = req.off;
     m.length = req.len;
     addMessage(m);

apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java

@@ -22,6 +22,7 @@ package org.klomp.snark;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Iterator;
@@ -34,6 +35,7 @@ import java.util.Set;
 import java.util.concurrent.LinkedBlockingQueue;
 
 import net.i2p.I2PAppContext;
+import net.i2p.data.ByteArray;
 import net.i2p.data.DataHelper;
 import net.i2p.data.Destination;
 import net.i2p.util.ConcurrentHashSet;
@@ -48,7 +50,7 @@ import org.klomp.snark.dht.DHT;
 /**
  * Coordinates what peer does what.
  */
-public class PeerCoordinator implements PeerListener
+class PeerCoordinator implements PeerListener
 {
   private final Log _log = I2PAppContext.getGlobalContext().logManager().getLog(PeerCoordinator.class);
 
@@ -68,6 +70,7 @@ public class PeerCoordinator implements PeerListener
   // package local for access by CheckDownLoadersTask
   final static long CHECK_PERIOD = 40*1000; // 40 seconds
   final static int MAX_UPLOADERS = 6;
+  public static final long MAX_INACTIVE = 8*60*1000;
 
   /**
    * Approximation of the number of current uploaders.
@@ -116,15 +119,21 @@ public class PeerCoordinator implements PeerListener
    */
   private final List<Piece> wantedPieces;
 
-  /** partial pieces - lock by synching on wantedPieces */
+  /** The total number of bytes in wantedPieces, or -1 if not yet known.
+   *  Sync on wantedPieces.
+   *  @since 0.9.1
+   */
+  private long wantedBytes;
+
+  /** partial pieces - lock by synching on wantedPieces - TODO store Requests, not PartialPieces */
   private final List<PartialPiece> partialPieces;
 
-  private boolean halted = false;
+  private volatile boolean halted;
 
   private final MagnetState magnetState;
   private final CoordinatorListener listener;
   private final I2PSnarkUtil _util;
-  private static final Random _random = I2PAppContext.getGlobalContext().random();
+  private final Random _random;
   
   /**
    *  @param metainfo null if in magnet mode
@@ -134,6 +143,7 @@ public class PeerCoordinator implements PeerListener
                          CoordinatorListener listener, Snark torrent)
   {
     _util = util;
+    _random = util.getContext().random();
     this.id = id;
     this.infohash = infohash;
     this.metainfo = metainfo;
@@ -151,7 +161,7 @@ public class PeerCoordinator implements PeerListener
     // Install a timer to check the uploaders.
     // Randomize the first start time so multiple tasks are spread out,
     // this will help the behavior with global limits
-    timer = new CheckEvent(new PeerCheckerTask(_util, this));
+    timer = new CheckEvent(_util.getContext(), new PeerCheckerTask(_util, this));
     timer.schedule((CHECK_PERIOD / 2) + _random.nextInt((int) CHECK_PERIOD));
   }
   
@@ -161,8 +171,8 @@ public class PeerCoordinator implements PeerListener
    */
   private static class CheckEvent extends SimpleTimer2.TimedEvent {
       private final PeerCheckerTask _task;
-      public CheckEvent(PeerCheckerTask task) {
-          super(SimpleTimer2.getInstance());
+      public CheckEvent(I2PAppContext ctx, PeerCheckerTask task) {
+          super(ctx.simpleTimer2());
           _task = task;
       }
       public void timeReached() {
@@ -171,16 +181,22 @@ public class PeerCoordinator implements PeerListener
       }
   }
 
-  // only called externally from Storage after the double-check fails
+  /**
+   * Only called externally from Storage after the double-check fails.
+   * Sets wantedBytes too.
+   */
   public void setWantedPieces()
   {
-    if (metainfo == null || storage == null)
+    if (metainfo == null || storage == null) {
+        wantedBytes = -1;
         return;
+    }
     // Make a list of pieces
       synchronized(wantedPieces) {
           wantedPieces.clear();
           BitField bitfield = storage.getBitField();
           int[] pri = storage.getPiecePriorities();
+          long count = 0;
           for (int i = 0; i < metainfo.getPieces(); i++) {
               // only add if we don't have and the priority is >= 0
               if ((!bitfield.get(i)) &&
@@ -189,15 +205,17 @@ public class PeerCoordinator implements PeerListener
                   if (pri != null)
                       p.setPriority(pri[i]);
                   wantedPieces.add(p);
+                  count += metainfo.getPieceLength(i);
               }
           }
+          wantedBytes = count;
           Collections.shuffle(wantedPieces, _random);
       }
   }
 
   public Storage getStorage() { return storage; }
 
-  // for web page detailed stats
+  /** for web page detailed stats */
   public List<Peer> peerList()
   {
         return new ArrayList(peers);
@@ -233,7 +251,9 @@ public class PeerCoordinator implements PeerListener
   }
 
   /**
-   * Returns how many bytes are still needed to get the complete file.
+   *  Bytes not yet in storage. Does NOT account for skipped files.
+   *  Not exact (does not adjust for last piece size).
+   * Returns how many bytes are still needed to get the complete torrent.
    * @return -1 if in magnet mode
    */
   public long getLeft()
@@ -244,6 +264,15 @@ public class PeerCoordinator implements PeerListener
     return ((long) storage.needed()) * metainfo.getPieceLength(0);
   }
 
+  /**
+   *  Bytes still wanted. DOES account for skipped files.
+   *  @return exact value. or -1 if no storage yet.
+   *  @since 0.9.1
+   */
+  public long getNeededLength() {
+      return wantedBytes;
+  }
+
   /**
    * Returns the total number of uploaded bytes of all peers.
    */
@@ -330,14 +359,35 @@ public class PeerCoordinator implements PeerListener
     return infohash;
   }
 
+  /**
+   *  Inbound.
+   *  Not halted, peers < max.
+   *  @since 0.9.1
+   */
   public boolean needPeers()
   {
         return !halted && peers.size() < getMaxConnections();
   }
+
+  /**
+   *  Outbound.
+   *  Not halted, peers < max, and need pieces.
+   *  @since 0.9.1
+   */
+  public boolean needOutboundPeers() {
+        //return wantedBytes != 0 && needPeers();
+        // minus one to make it a little easier for new peers to get in on large swarms
+        return wantedBytes != 0 &&
+               !halted &&
+               peers.size() < getMaxConnections() - 1 &&
+               (storage == null || !storage.isChecking());
+  }
   
   /**
-   *  Reduce max if huge pieces to keep from ooming when leeching
-   *  @return 512K: 16; 1M: 11; 2M: 6
+   *  Formerly used to
+   *  reduce max if huge pieces to keep from ooming when leeching
+   *  but now we don't
+   *  @return usually 16	
    */
   private int getMaxConnections() {
     if (metainfo == null)
@@ -347,13 +397,14 @@ public class PeerCoordinator implements PeerListener
         return 4;
     if (pieces <= 5)
         return 6;
-    int size = metainfo.getPieceLength(0);
+    //int size = metainfo.getPieceLength(0);
     int max = _util.getMaxConnections();
-    if (size <= 512*1024 || completed())
+    // Now that we use temp files, no memory concern
+    //if (size <= 512*1024 || completed())
       return max;
-    if (size <= 1024*1024)
-      return (max + max + 2) / 3;
-    return (max + 2) / 3;
+    //if (size <= 1024*1024)
+    //  return (max + max + 2) / 3;
+    //return (max + 2) / 3;
   }
 
   public boolean halted() { return halted; }
@@ -380,10 +431,33 @@ public class PeerCoordinator implements PeerListener
     }
     // delete any saved orphan partial piece
     synchronized (partialPieces) {
+        for (PartialPiece pp : partialPieces) {
+            pp.release();
+        }
         partialPieces.clear();
     }
   }
 
+  /**
+   *  @since 0.9.1
+   */
+  public void restart() {
+    halted = false;
+    synchronized (uploaded_old) {
+        Arrays.fill(uploaded_old, 0);
+    }
+    synchronized (downloaded_old) {
+        Arrays.fill(downloaded_old, 0);
+    }
+    // failsafe
+    synchronized(wantedPieces) {
+        for (Piece pc : wantedPieces) {
+            pc.clear();
+        }
+    }
+    timer.schedule((CHECK_PERIOD / 2) + _random.nextInt((int) CHECK_PERIOD));
+  }
+
   public void connected(Peer peer)
   { 
     if (halted)
@@ -396,7 +470,7 @@ public class PeerCoordinator implements PeerListener
     synchronized(peers)
       {
         Peer old = peerIDInList(peer.getPeerID(), peers);
-        if ( (old != null) && (old.getInactiveTime() > 8*60*1000) ) {
+        if ( (old != null) && (old.getInactiveTime() > MAX_INACTIVE) ) {
             // idle for 8 minutes, kill the old con (32KB/8min = 68B/sec minimum for one block)
             if (_log.shouldLog(Log.WARN))
               _log.warn("Remomving old peer: " + peer + ": " + old + ", inactive for " + old.getInactiveTime());
@@ -444,8 +518,8 @@ public class PeerCoordinator implements PeerListener
             peerCount = peers.size();
             unchokePeer();
 
-            if (listener != null)
-              listener.peerChange(this, peer);
+            //if (listener != null)
+            //  listener.peerChange(this, peer);
           }
       }
     if (toDisconnect != null) {
@@ -468,7 +542,10 @@ public class PeerCoordinator implements PeerListener
     return null;
   }
 
-// returns true if actual attempt to add peer occurs
+  /**
+   * Add peer (inbound or outbound)
+   * @return true if actual attempt to add peer occurs
+   */
   public boolean addPeer(final Peer peer)
   {
     if (halted)
@@ -487,7 +564,7 @@ public class PeerCoordinator implements PeerListener
         need_more = (!peer.isConnected()) && peersize < getMaxConnections();
         // Check if we already have this peer before we build the connection
         Peer old = peerIDInList(peer.getPeerID(), peers);
-        need_more = need_more && ((old == null) || (old.getInactiveTime() > 8*60*1000));
+        need_more = need_more && ((old == null) || (old.getInactiveTime() > MAX_INACTIVE));
       }
 
     if (need_more)
@@ -578,13 +655,18 @@ public class PeerCoordinator implements PeerListener
    */
   public boolean gotHave(Peer peer, int piece)
   {
-    if (listener != null)
-      listener.peerChange(this, peer);
+    //if (listener != null)
+    //  listener.peerChange(this, peer);
 
-    synchronized(wantedPieces)
-      {
-        return wantedPieces.contains(new Piece(piece));
-      }
+    synchronized(wantedPieces) {
+        for (Piece pc : wantedPieces) {
+            if (pc.getId() == piece) {
+                pc.addPeer(peer);
+                return true;
+            }
+        }
+        return false;
+    }
   }
 
   /**
@@ -593,23 +675,20 @@ public class PeerCoordinator implements PeerListener
    */
   public boolean gotBitField(Peer peer, BitField bitfield)
   {
-    if (listener != null)
-      listener.peerChange(this, peer);
+    //if (listener != null)
+    //  listener.peerChange(this, peer);
 
-    synchronized(wantedPieces)
-      {
-        Iterator<Piece> it = wantedPieces.iterator();
-        while (it.hasNext())
-          {
-            Piece p = it.next();
+    boolean rv = false;
+    synchronized(wantedPieces) {
+        for (Piece p : wantedPieces) {
             int i = p.getId();
             if (bitfield.get(i)) {
               p.addPeer(peer);
-              return true;
+              rv = true;
             }
-          }
-      }
-    return false;
+        }
+    }
+    return rv;
   }
 
   /**
@@ -630,22 +709,23 @@ public class PeerCoordinator implements PeerListener
    * -1 if none of the given pieces are wanted.
    */
   public int wantPiece(Peer peer, BitField havePieces) {
-      return wantPiece(peer, havePieces, true);
+      Piece pc = wantPiece(peer, havePieces, true);
+      return pc != null ? pc.getId() : -1;
   }
 
   /**
    * Returns one of pieces in the given BitField that is still wanted or
-   * -1 if none of the given pieces are wanted.
+   * null if none of the given pieces are wanted.
    *
    * @param record if true, actually record in our data structures that we gave the
    *               request to this peer. If false, do not update the data structures.
    * @since 0.8.2
    */
-  private int wantPiece(Peer peer, BitField havePieces, boolean record) {
+  private Piece wantPiece(Peer peer, BitField havePieces, boolean record) {
     if (halted) {
       if (_log.shouldLog(Log.WARN))
           _log.warn("We don't want anything from the peer, as we are halted!  peer=" + peer);
-      return -1;
+      return null;
     }
 
     Piece piece = null;
@@ -664,7 +744,19 @@ public class PeerCoordinator implements PeerListener
                 break;
             if (havePieces.get(p.getId()) && !p.isRequested())
               {
-                piece = p;
+                // never ever choose one that's in partialPieces, or we
+                // will create a second one and leak
+                boolean hasPartial = false;
+                for (PartialPiece pp : partialPieces) {
+                    if (pp.getPiece() == p.getId()) {
+                        if (_log.shouldLog(Log.INFO))
+                            _log.info("wantPiece() skipping partial for " + peer + ": piece = " + pp);
+                        hasPartial = true;
+                        break;
+                    }
+                }
+                if (!hasPartial)
+                    piece = p;
               }
             else if (p.isRequested()) 
             {
@@ -679,8 +771,12 @@ public class PeerCoordinator implements PeerListener
             // AND if there are almost no wanted pieces left (real end game).
             // If we do end game all the time, we generate lots of extra traffic
             // when the seeder is super-slow and all the peers are "caught up"
-            if (wantedSize > END_GAME_THRESHOLD)
-                return -1;  // nothing to request and not in end game
+            if (wantedSize > END_GAME_THRESHOLD) {
+                if (_log.shouldLog(Log.INFO))
+                    _log.info("Nothing to request, " + requested.size() + " being requested and " +
+                              wantedSize + " still wanted");
+                return null;  // nothing to request and not in end game
+            }
             // let's not all get on the same piece
             // Even better would be to sort by number of requests
             if (record)
@@ -704,7 +800,7 @@ public class PeerCoordinator implements PeerListener
                     _log.warn("nothing to even rerequest from " + peer + ": requested = " + requested);
                 //  _log.warn("nothing to even rerequest from " + peer + ": requested = " + requested 
                 //            + " wanted = " + wantedPieces + " peerHas = " + havePieces);
-                return -1; //If we still can't find a piece we want, so be it.
+                return null; //If we still can't find a piece we want, so be it.
             } else {
                 // Should be a lot smarter here -
                 // share blocks rather than starting from 0 with each peer.
@@ -716,10 +812,11 @@ public class PeerCoordinator implements PeerListener
         }
         if (record) {
             if (_log.shouldLog(Log.INFO))
-                _log.info(peer + " is now requesting: piece " + piece + " priority " + piece.getPriority());
+                _log.info("Now requesting from " + peer + ": piece " + piece + " priority " + piece.getPriority() +
+                          " peers " + piece.getPeerCount() + '/' + peers.size());
             piece.setRequested(peer, true);
         }
-        return piece.getId();
+        return piece;
       } // synch
   }
 
@@ -736,6 +833,7 @@ public class PeerCoordinator implements PeerListener
           _log.debug("Updated piece priorities called but no priorities to set?");
           return;
       }
+      List<Piece> toCancel = new ArrayList();
       synchronized(wantedPieces) {
           // Add incomplete and previously unwanted pieces to the list
           // Temp to avoid O(n**2)
@@ -749,6 +847,7 @@ public class PeerCoordinator implements PeerListener
                   if (!want.get(i)) {
                       Piece piece = new Piece(i);
                       wantedPieces.add(piece);
+                      wantedBytes += metainfo.getPieceLength(i);
                       // As connections are already up, new Pieces will
                       // not have their PeerID list populated, so do that.
                           for (Peer p : peers) {
@@ -770,23 +869,32 @@ public class PeerCoordinator implements PeerListener
                    p.setPriority(priority);
                } else {
                    iter.remove();
-                   // cancel all peers
-                       for (Peer peer : peers) {
-                           peer.cancel(p.getId());
-                       }
+                   toCancel.add(p);
+                   wantedBytes -= metainfo.getPieceLength(p.getId());
                }
           }
           if (_log.shouldLog(Log.DEBUG))
               _log.debug("Updated piece priorities, now wanted: " + wantedPieces);
           // if we added pieces, they will be in-order unless we shuffle
           Collections.shuffle(wantedPieces, _random);
+      }
 
-          // update request queues, in case we added wanted pieces
-          // and we were previously uninterested
-              for (Peer peer : peers) {
-                  peer.request();
+      // cancel outside of wantedPieces lock to avoid deadlocks
+      if (!toCancel.isEmpty()) {
+          // cancel all peers
+          for (Peer peer : peers) {
+              for (Piece p : toCancel) {
+                  peer.cancel(p.getId());
+              }
           }
       }
+
+      // ditto, avoid deadlocks
+      // update request queues, in case we added wanted pieces
+      // and we were previously uninterested
+      for (Peer peer : peers) {
+          peer.request();
+      }
   }
 
   /**
@@ -795,7 +903,7 @@ public class PeerCoordinator implements PeerListener
    *
    * @throws RuntimeException on IOE getting the data
    */
-  public byte[] gotRequest(Peer peer, int piece, int off, int len)
+  public ByteArray gotRequest(Peer peer, int piece, int off, int len)
   {
     if (halted)
       return null;
@@ -811,8 +919,10 @@ public class PeerCoordinator implements PeerListener
         snark.stopTorrent();
         String msg = "Error reading the storage (piece " + piece + ") for " + metainfo.getName() + ": " + ioe;
         _log.error(msg, ioe);
-        SnarkManager.instance().addMessage(msg);
-        SnarkManager.instance().addMessage("Fatal storage error: Stopping torrent " + metainfo.getName());
+        if (listener != null) {
+            listener.addMessage(msg);
+            listener.addMessage("Fatal storage error: Stopping torrent " + metainfo.getName());
+        }
         throw new RuntimeException(msg, ioe);
       }
   }
@@ -824,8 +934,8 @@ public class PeerCoordinator implements PeerListener
   {
     uploaded += size;
 
-    if (listener != null)
-      listener.peerChange(this, peer);
+    //if (listener != null)
+    //  listener.peerChange(this, peer);
   }
 
   /**
@@ -835,8 +945,8 @@ public class PeerCoordinator implements PeerListener
   {
     downloaded += size;
 
-    if (listener != null)
-      listener.peerChange(this, peer);
+    //if (listener != null)
+    //  listener.peerChange(this, peer);
   }
 
   /**
@@ -846,14 +956,13 @@ public class PeerCoordinator implements PeerListener
    *
    * @throws RuntimeException on IOE saving the piece
    */
-  public boolean gotPiece(Peer peer, int piece, byte[] bs)
+  public boolean gotPiece(Peer peer, PartialPiece pp)
   {
-    if (metainfo == null || storage == null)
+    if (metainfo == null || storage == null || storage.isChecking() || halted) {
+        pp.release();
         return true;
-    if (halted) {
-      _log.info("Got while-halted piece " + piece + "/" + metainfo.getPieces() +" from " + peer + " for " + metainfo.getName());
-      return true; // We don't actually care anymore.
     }
+    int piece = pp.getPiece();
     
     synchronized(wantedPieces)
       {
@@ -866,13 +975,16 @@ public class PeerCoordinator implements PeerListener
             // Assume we got a good piece, we don't really care anymore.
             // Well, this could be caused by a change in priorities, so
             // only return true if we already have it, otherwise might as well keep it.
-            if (storage.getBitField().get(piece))
+            if (storage.getBitField().get(piece)) {
+                pp.release();
                 return true;
+            }
           }
         
         try
           {
-            if (storage.putPiece(piece, bs))
+            // this takes forever if complete, as it rechecks
+            if (storage.putPiece(pp))
               {
                 if (_log.shouldLog(Log.INFO))
                     _log.info("Got valid piece " + piece + "/" + metainfo.getPieces() +" from " + peer + " for " + metainfo.getName());
@@ -890,38 +1002,51 @@ public class PeerCoordinator implements PeerListener
             snark.stopTorrent();
             String msg = "Error writing storage (piece " + piece + ") for " + metainfo.getName() + ": " + ioe;
             _log.error(msg, ioe);
-            SnarkManager.instance().addMessage(msg);
-            SnarkManager.instance().addMessage("Fatal storage error: Stopping torrent " + metainfo.getName());
+            if (listener != null) {
+                listener.addMessage(msg);
+                listener.addMessage("Fatal storage error: Stopping torrent " + metainfo.getName());
+            }
             throw new RuntimeException(msg, ioe);
           }
         wantedPieces.remove(p);
+        wantedBytes -= metainfo.getPieceLength(p.getId());
       }
 
     // just in case
     removePartialPiece(piece);
 
+    boolean done = wantedBytes <= 0;
+
     // Announce to the world we have it!
     // Disconnect from other seeders when we get the last piece
-        List<Peer> toDisconnect = new ArrayList(); 
-        Iterator<Peer> it = peers.iterator();
-        while (it.hasNext())
-          {
-            Peer p = it.next();
+    List<Peer> toDisconnect = done ? new ArrayList() : null; 
+    for (Peer p : peers) {
             if (p.isConnected())
               {
-                  if (completed() && p.isCompleted())
+                  if (done && p.isCompleted())
                       toDisconnect.add(p);
                   else
                       p.have(piece);
               }
-          }
-        it = toDisconnect.iterator();
-        while (it.hasNext())
-          {
-            Peer p = it.next();
+    }
+
+    if (done) {
+        for (Peer p : toDisconnect) {
             p.disconnect(true);
-          }
+        }
     
+        // put msg on the console if partial, since Storage won't do it
+        if (!completed())
+            snark.storageCompleted(storage);
+
+        synchronized (partialPieces) {
+            for (PartialPiece ppp : partialPieces) {
+                ppp.release();
+            }
+            partialPieces.clear();
+        }
+    }
+
     return true;
   }
 
@@ -931,8 +1056,8 @@ public class PeerCoordinator implements PeerListener
     if (_log.shouldLog(Log.INFO))
       _log.info("Got choke(" + choke + "): " + peer);
 
-    if (listener != null)
-      listener.peerChange(this, peer);
+    //if (listener != null)
+    //  listener.peerChange(this, peer);
   }
 
   public void gotInterest(Peer peer, boolean interest)
@@ -951,8 +1076,8 @@ public class PeerCoordinator implements PeerListener
               }
       }
 
-    if (listener != null)
-      listener.peerChange(this, peer);
+    //if (listener != null)
+    //  listener.peerChange(this, peer);
   }
 
   public void disconnected(Peer peer)
@@ -972,18 +1097,18 @@ public class PeerCoordinator implements PeerListener
         peerCount = peers.size();
       }
 
-    if (listener != null)
-      listener.peerChange(this, peer);
+    //if (listener != null)
+    //  listener.peerChange(this, peer);
   }
   
   /** Called when a peer is removed, to prevent it from being used in 
    * rarest-first calculations.
    */
-  public void removePeerFromPieces(Peer peer) {
+  private void removePeerFromPieces(Peer peer) {
       synchronized(wantedPieces) {
-          for(Iterator<Piece> iter = wantedPieces.iterator(); iter.hasNext(); ) {
-              Piece piece = iter.next();
+          for (Piece piece : wantedPieces) {
               piece.removePeer(peer);
+              piece.setRequested(peer, false);
           }
       } 
   }
@@ -998,17 +1123,24 @@ public class PeerCoordinator implements PeerListener
    *  Also mark the piece unrequested if this peer was the only one.
    *
    *  @param peer partials, must include the zero-offset (empty) ones too
+   *              No dup pieces, piece.setDownloaded() must be set
    *  @since 0.8.2
    */
-  public void savePartialPieces(Peer peer, List<PartialPiece> partials)
+  public void savePartialPieces(Peer peer, List<Request> partials)
   {
-      if (halted)
-          return;
       if (_log.shouldLog(Log.INFO))
           _log.info("Partials received from " + peer + ": " + partials);
+      if (halted || completed()) {
+          for (Request req : partials) {
+              PartialPiece pp = req.getPartialPiece();
+              pp.release();
+          }
+          return;
+      }
       synchronized(wantedPieces) {
-          for (PartialPiece pp : partials) {
-              if (pp.getDownloaded() > 0) {
+          for (Request req : partials) {
+              PartialPiece pp = req.getPartialPiece();
+              if (req.off > 0) {
                   // PartialPiece.equals() only compares piece number, which is what we want
                   int idx = partialPieces.indexOf(pp);
                   if (idx < 0) {
@@ -1017,10 +1149,12 @@ public class PeerCoordinator implements PeerListener
                           _log.info("Saving orphaned partial piece (new) " + pp);
                   } else if (idx >= 0 && pp.getDownloaded() > partialPieces.get(idx).getDownloaded()) {
                       // replace what's there now
+                      partialPieces.get(idx).release();
                       partialPieces.set(idx, pp);
                       if (_log.shouldLog(Log.INFO))
                           _log.info("Saving orphaned partial piece (bigger) " + pp);
                   } else {
+                      pp.release();
                       if (_log.shouldLog(Log.INFO))
                           _log.info("Discarding partial piece (not bigger)" + pp);
                   }
@@ -1029,10 +1163,14 @@ public class PeerCoordinator implements PeerListener
                       // sorts by remaining bytes, least first
                       Collections.sort(partialPieces);
                       PartialPiece gone = partialPieces.remove(max);
+                      gone.release();
                       if (_log.shouldLog(Log.INFO))
                           _log.info("Discarding orphaned partial piece (list full)" + gone);
                   }
-              }  // else drop the empty partial piece
+              } else {
+                  // drop the empty partial piece
+                  pp.release();
+              }
               // synchs on wantedPieces...
               markUnrequested(peer, pp.getPiece());
           }
@@ -1051,6 +1189,8 @@ public class PeerCoordinator implements PeerListener
   public PartialPiece getPartialPiece(Peer peer, BitField havePieces) {
       if (metainfo == null)
           return null;
+      if (storage != null && storage.isChecking())
+          return null;
       synchronized(wantedPieces) {
           // sorts by remaining bytes, least first
           Collections.sort(partialPieces);
@@ -1058,10 +1198,31 @@ public class PeerCoordinator implements PeerListener
               PartialPiece pp = iter.next();
               int savedPiece = pp.getPiece();
               if (havePieces.get(savedPiece)) {
-                 iter.remove();
                  // this is just a double-check, it should be in there
+                 boolean skipped = false;
                  for(Piece piece : wantedPieces) {
                      if (piece.getId() == savedPiece) {
+                         if (peer.isCompleted() && piece.getPeerCount() > 1) {
+                             // Try to preserve rarest-first
+                             // by not requesting a partial piece that non-seeders also have
+                             // from a seeder
+                             boolean nonSeeds = false;
+                             for (Peer pr : peers) {
+                                 PeerState state = pr.state;
+                                 if (state == null) continue;
+                                 BitField bf = state.bitfield;
+                                 if (bf == null) continue;
+                                 if (bf.get(savedPiece) && !pr.isCompleted()) {
+                                     nonSeeds = true;
+                                     break;
+                                 }
+                             }
+                             if (nonSeeds) {
+                                 skipped = true;
+                                 break;
+                             }
+                         }
+                         iter.remove();
                          piece.setRequested(peer, true);
                          if (_log.shouldLog(Log.INFO)) {
                              _log.info("Restoring orphaned partial piece " + pp +
@@ -1070,8 +1231,12 @@ public class PeerCoordinator implements PeerListener
                          return pp;
                       }
                   }
-                  if (_log.shouldLog(Log.WARN))
-                      _log.warn("Partial piece " + pp + " NOT in wantedPieces??");
+                  if (_log.shouldLog(Log.WARN)) {
+                      if (skipped)
+                          _log.warn("Partial piece " + pp + " with multiple peers skipped for seeder");
+                      else
+                          _log.warn("Partial piece " + pp + " NOT in wantedPieces??");
+                  }
               }
           }
           if (_log.shouldLog(Log.WARN) && !partialPieces.isEmpty())
@@ -1079,14 +1244,9 @@ public class PeerCoordinator implements PeerListener
       }
       // ...and this section turns this into the general move-requests-around code!
       // Temporary? So PeerState never calls wantPiece() directly for now...
-      int piece = wantPiece(peer, havePieces);
-      if (piece >= 0) {
-          try {
-              return new PartialPiece(piece, metainfo.getPieceLength(piece));
-          } catch (OutOfMemoryError oom) {
-              if (_log.shouldLog(Log.WARN))
-                  _log.warn("OOM creating new partial piece");
-          }
+      Piece piece = wantPiece(peer, havePieces, true);
+      if (piece != null) {
+          return new PartialPiece(piece, metainfo.getPieceLength(piece.getId()), _util.getTempDir());
       }
       if (_log.shouldLog(Log.DEBUG))
           _log.debug("We have no partial piece to return");
@@ -1121,7 +1281,7 @@ public class PeerCoordinator implements PeerListener
               }
           }
       }
-      return wantPiece(peer, havePieces, false) >= 0;
+      return wantPiece(peer, havePieces, false) != null;
   }
 
   /**
@@ -1135,6 +1295,7 @@ public class PeerCoordinator implements PeerListener
               PartialPiece pp = iter.next();
               if (pp.getPiece() == piece) {
                   iter.remove();
+                  pp.release();
                   // there should be only one but keep going to be sure
               }
           }
@@ -1176,6 +1337,7 @@ public class PeerCoordinator implements PeerListener
           }
       } else if (id == ExtensionHandler.ID_HANDSHAKE) {
           sendPeers(peer);
+          sendDHT(peer);
       }
   }
 
@@ -1186,6 +1348,8 @@ public class PeerCoordinator implements PeerListener
    *  @since 0.8.4
    */
   void sendPeers(Peer peer) {
+      if (metainfo != null && metainfo.isPrivate())
+          return;
       Map<String, BEValue> handshake = peer.getHandshakeMap();
       if (handshake == null)
           return;
@@ -1202,6 +1366,26 @@ public class PeerCoordinator implements PeerListener
       } catch (InvalidBEncodingException ibee) {}
   }
 
+  /**
+   *  Send a DHT message to the peer, if we both support DHT.
+   *  @since DHT
+   */
+  void sendDHT(Peer peer) {
+      DHT dht = _util.getDHT();
+      if (dht == null)
+          return;
+      Map<String, BEValue> handshake = peer.getHandshakeMap();
+      if (handshake == null)
+          return;
+      BEValue bev = handshake.get("m");
+      if (bev == null)
+          return;
+      try {
+          if (bev.getMap().get(ExtensionHandler.TYPE_DHT) != null)
+              ExtensionHandler.sendDHT(peer, dht.getPort(), dht.getRPort());
+      } catch (InvalidBEncodingException ibee) {}
+  }
+
   /**
    *  Sets the storage after transition out of magnet mode
    *  Snark calls this after we call gotMetaInfo()
@@ -1219,20 +1403,23 @@ public class PeerCoordinator implements PeerListener
   /**
    *  PeerListener callback
    *  Tell the DHT to ping it, this will get back the node info
+   *  @param rport must be port + 1
    *  @since 0.8.4
    */
-  public void gotPort(Peer peer, int port) {
+  public void gotPort(Peer peer, int port, int rport) {
       DHT dht = _util.getDHT();
-      if (dht != null)
+      if (dht != null &&
+          port > 0 && port < 65535 && rport == port + 1)
           dht.ping(peer.getDestination(), port);
   }
 
   /**
+   *  Get peers from PEX -
    *  PeerListener callback
    *  @since 0.8.4
    */
   public void gotPeers(Peer peer, List<PeerID> peers) {
-      if (completed() || !needPeers())
+      if (!needOutboundPeers())
           return;
       Destination myDest = _util.getMyDestination();
       if (myDest == null)
@@ -1252,6 +1439,7 @@ public class PeerCoordinator implements PeerListener
 
   /**
    *  Called by TrackerClient
+   *  @return the Set itself, modifiable, not a copy, caller should clear()
    *  @since 0.8.4
    */
   Set<PeerID> getPEXPeers() {
@@ -1294,5 +1482,13 @@ public class PeerCoordinator implements PeerListener
         return listener.overUpBWLimit(total * 1000 / CHECK_PERIOD);
     return false;
   }
+
+  /**
+   *  Convenience
+   *  @since 0.9.2
+   */
+  public I2PSnarkUtil getUtil() {
+      return _util;
+  }
 }
 

apps/i2psnark/java/src/org/klomp/snark/PeerCoordinatorSet.java

@@ -1,9 +1,10 @@
 package org.klomp.snark;
 
-import java.util.ArrayList;
-import java.util.HashSet;
 import java.util.Iterator;
-import java.util.Set;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import net.i2p.crypto.SHA1Hash;
 
 /**
  * Hmm, any guesses as to what this is?  Used by the multitorrent functionality
@@ -12,26 +13,28 @@ import java.util.Set;
  * from it there too)
  */
 public class PeerCoordinatorSet {
-    private final Set _coordinators;
+    private final Map<SHA1Hash, PeerCoordinator> _coordinators;
     
     public PeerCoordinatorSet() {
-        _coordinators = new HashSet();
+        _coordinators = new ConcurrentHashMap();
     }
-    
-    public Iterator iterator() {
-        synchronized (_coordinators) {
-            return new ArrayList(_coordinators).iterator();
-        }
+     
+    public Iterator<PeerCoordinator> iterator() {
+        return _coordinators.values().iterator();
     }
-    
+
     public void add(PeerCoordinator coordinator) {
-        synchronized (_coordinators) {
-            _coordinators.add(coordinator);
-        }
+        _coordinators.put(new SHA1Hash(coordinator.getInfoHash()), coordinator);
     }
+
     public void remove(PeerCoordinator coordinator) {
-        synchronized (_coordinators) {
-            _coordinators.remove(coordinator);
-        }
+        _coordinators.remove(new SHA1Hash(coordinator.getInfoHash()));
+    }
+
+    /**
+     *  @since 0.9.2
+     */
+    public PeerCoordinator get(byte[] infoHash) {
+        return _coordinators.get(new SHA1Hash(infoHash));
     }
 }

apps/i2psnark/java/src/org/klomp/snark/PeerID.java

@@ -52,6 +52,7 @@ public class PeerID implements Comparable
   /** whether we have tried to get the dest from the hash - only do once */
   private boolean triedDestLookup;
   private final int hash;
+  private final I2PSnarkUtil util;
 
   public PeerID(byte[] id, Destination address)
   {
@@ -60,6 +61,7 @@ public class PeerID implements Comparable
     this.port = 6881;
     this.destHash = address.calculateHash().getData();
     hash = calculateHash();
+    util = null;
   }
 
   /**
@@ -93,13 +95,15 @@ public class PeerID implements Comparable
     port = 6881;
     this.destHash = address.calculateHash().getData();
     hash = calculateHash();
+    util = null;
   }
 
   /**
    * Creates a PeerID from a destHash
+   * @param util for eventual destination lookup
    * @since 0.8.1
    */
-  public PeerID(byte[] dest_hash) throws InvalidBEncodingException
+  public PeerID(byte[] dest_hash, I2PSnarkUtil util) throws InvalidBEncodingException
   {
     // id and address remain null
     port = 6881;
@@ -107,6 +111,7 @@ public class PeerID implements Comparable
         throw new InvalidBEncodingException("bad hash length");
     destHash = dest_hash;
     hash = DataHelper.hashCode(dest_hash);
+    this.util = util;
   }
 
   public byte[] getID()
@@ -131,7 +136,7 @@ public class PeerID implements Comparable
   {
     if (address == null && destHash != null && !triedDestLookup) {
         String b32 = Base32.encode(destHash) + ".b32.i2p";
-        address = I2PAppContext.getGlobalContext().namingService().lookup(b32);
+        address = util.getDestination(b32);
         triedDestLookup = true;
     }
     return address;

apps/i2psnark/java/src/org/klomp/snark/PeerListener.java

@@ -22,6 +22,8 @@ package org.klomp.snark;
 
 import java.util.List;
 
+import net.i2p.data.ByteArray;
+
 /**
  * Listener for Peer events.
  */
@@ -95,12 +97,11 @@ interface PeerListener
    * will be closed.
    *
    * @param peer the Peer that got the piece.
-   * @param piece the piece number received.
-   * @param bs the byte array containing the piece.
+   * @param piece the piece received.
    *
    * @return true when the bytes represent the piece, false otherwise.
    */
-  boolean gotPiece(Peer peer, int piece, byte[] bs);
+  boolean gotPiece(Peer peer, PartialPiece piece);
 
   /**
    * Called when the peer wants (part of) a piece from us. Only called
@@ -115,7 +116,7 @@ interface PeerListener
    * @return a byte array containing the piece or null when the piece
    * is not available (which is a protocol error).
    */
-  byte[] gotRequest(Peer peer, int piece, int off, int len);
+  ByteArray gotRequest(Peer peer, int piece, int off, int len);
 
   /**
    * Called when a (partial) piece has been downloaded from the peer.
@@ -167,7 +168,7 @@ interface PeerListener
    * @param peer the peer
    * @since 0.8.2
    */
-  void savePartialPieces(Peer peer, List<PartialPiece> pcs);
+  void savePartialPieces(Peer peer, List<Request> pcs);
 
   /**
    * Called when a peer has connected and there may be a partially
@@ -191,13 +192,14 @@ interface PeerListener
   void gotExtension(Peer peer, int id, byte[] bs);
 
   /**
-   * Called when a port message is received.
+   * Called when a DHT port message is received.
    *
    * @param peer the Peer that got the message.
-   * @param port the port
+   * @param port the query port
+   * @param rport the response port
    * @since 0.8.4
    */
-  void gotPort(Peer peer, int port);
+  void gotPort(Peer peer, int port, int rport);
 
   /**
    * Called when peers are received via PEX
@@ -207,4 +209,10 @@ interface PeerListener
    * @since 0.8.4
    */
   void gotPeers(Peer peer, List<PeerID> pIDList);
+
+  /**
+   *  Convenience
+   *  @since 0.9.2
+   */
+  public I2PSnarkUtil getUtil();
 }

apps/i2psnark/java/src/org/klomp/snark/PeerState.java

@@ -20,16 +20,14 @@
 
 package org.klomp.snark;
 
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
 import net.i2p.I2PAppContext;
+import net.i2p.data.ByteArray;
 import net.i2p.util.Log;
 
 class PeerState implements DataLoader
@@ -42,13 +40,13 @@ class PeerState implements DataLoader
 
   // Interesting and choking describes whether we are interested in or
   // are choking the other side.
-  boolean interesting = false;
-  boolean choking = true;
+  volatile boolean interesting;
+  volatile boolean choking = true;
 
   // Interested and choked describes whether the other side is
   // interested in us or choked us.
-  boolean interested = false;
-  boolean choked = true;
+  volatile boolean interested;
+  volatile boolean choked = true;
 
   /** the pieces the peer has */
   BitField bitfield;
@@ -110,7 +108,7 @@ class PeerState implements DataLoader
         // The only problem with returning the partials to the coordinator
         // is that chunks above a missing request are lost.
         // Future enhancements to PartialPiece could keep track of the holes.
-        List<PartialPiece> pcs = returnPartialPieces();
+        List<Request> pcs = returnPartialPieces();
         if (!pcs.isEmpty()) {
             if (_log.shouldLog(Log.DEBUG))
                 _log.debug(peer + " got choked, returning partial pieces to the PeerCoordinator: " + pcs);
@@ -234,8 +232,8 @@ class PeerState implements DataLoader
         return;
       }
 
-    if (_log.shouldLog(Log.INFO))
-      _log.info("Queueing (" + piece + ", " + begin + ", "
+    if (_log.shouldLog(Log.DEBUG))
+        _log.debug("Queueing (" + piece + ", " + begin + ", "
                 + length + ")" + " to " + peer);
 
     // don't load the data into mem now, let PeerConnectionOut do it
@@ -248,8 +246,8 @@ class PeerState implements DataLoader
    *  @return bytes or null for errors
    *  @since 0.8.2
    */
-  public byte[] loadData(int piece, int begin, int length) {
-    byte[] pieceBytes = listener.gotRequest(peer, piece, begin, length);
+  public ByteArray loadData(int piece, int begin, int length) {
+    ByteArray pieceBytes = listener.gotRequest(peer, piece, begin, length);
     if (pieceBytes == null)
       {
         // XXX - Protocol error-> diconnect?
@@ -259,7 +257,7 @@ class PeerState implements DataLoader
       }
 
     // More sanity checks
-    if (length != pieceBytes.length)
+    if (length != pieceBytes.getData().length)
       {
         // XXX - Protocol error-> disconnect?
         if (_log.shouldLog(Log.WARN))
@@ -270,8 +268,8 @@ class PeerState implements DataLoader
         return null;
       }
 
-    if (_log.shouldLog(Log.INFO))
-      _log.info("Sending (" + piece + ", " + begin + ", "
+    if (_log.shouldLog(Log.DEBUG))
+        _log.debug("Sending (" + piece + ", " + begin + ", "
                 + length + ")" + " to " + peer);
     return pieceBytes;
   }
@@ -307,22 +305,23 @@ class PeerState implements DataLoader
 
     if (_log.shouldLog(Log.DEBUG))
       _log.debug("got end of Chunk("
-                  + req.piece + "," + req.off + "," + req.len + ") from "
+                  + req.getPiece() + "," + req.off + "," + req.len + ") from "
                   + peer);
 
     // Last chunk needed for this piece?
-    if (getFirstOutstandingRequest(req.piece) == -1)
+    // FIXME if priority changed to skip, we will think we're done when we aren't
+    if (getFirstOutstandingRequest(req.getPiece()) == -1)
       {
         // warning - may block here for a while
-        if (listener.gotPiece(peer, req.piece, req.bs))
+        if (listener.gotPiece(peer, req.getPartialPiece()))
           {
             if (_log.shouldLog(Log.DEBUG))
-              _log.debug("Got " + req.piece + ": " + peer);
+              _log.debug("Got " + req.getPiece() + ": " + peer);
           }
         else
           {
             if (_log.shouldLog(Log.WARN))
-              _log.warn("Got BAD " + req.piece + " from " + peer);
+              _log.warn("Got BAD " + req.getPiece() + " from " + peer);
           }
       }
 
@@ -338,7 +337,7 @@ class PeerState implements DataLoader
   synchronized private int getFirstOutstandingRequest(int piece)
    {
     for (int i = 0; i < outstandingRequests.size(); i++)
-      if (outstandingRequests.get(i).piece == piece)
+      if (outstandingRequests.get(i).getPiece() == piece)
         return i;
     return -1;
   }
@@ -357,24 +356,23 @@ class PeerState implements DataLoader
                   + piece + "," + begin + "," + length + ") from "
                   + peer);
 
-    int r = getFirstOutstandingRequest(piece);
-
-    // Unrequested piece number?
-    if (r == -1)
-      {
-        if (_log.shouldLog(Log.INFO))
-          _log.info("Unrequested 'piece: " + piece + ", "
-                      + begin + ", " + length + "' received from "
-                      + peer);
-        return null;
-      }
-
     // Lookup the correct piece chunk request from the list.
     Request req;
     synchronized(this)
       {
+        int r = getFirstOutstandingRequest(piece);
+
+        // Unrequested piece number?
+        if (r == -1) {
+            if (_log.shouldLog(Log.INFO))
+                _log.info("Unrequested 'piece: " + piece + ", "
+                      + begin + ", " + length + "' received from "
+                      + peer);
+            return null;
+        }
+
         req = outstandingRequests.get(r);
-        while (req.piece == piece && req.off != begin
+        while (req.getPiece() == piece && req.off != begin
                && r < outstandingRequests.size() - 1)
           {
             r++;
@@ -382,7 +380,7 @@ class PeerState implements DataLoader
           }
         
         // Something wrong?
-        if (req.piece != piece || req.off != begin || req.len != length)
+        if (req.getPiece() != piece || req.off != begin || req.len != length)
           {
             if (_log.shouldLog(Log.INFO))
               _log.info("Unrequested or unneeded 'piece: "
@@ -430,13 +428,13 @@ class PeerState implements DataLoader
       Request rv = null;
       int lowest = Integer.MAX_VALUE;
       for (Request r :  outstandingRequests) {
-          if (r.piece == piece && r.off < lowest) {
+          if (r.getPiece() == piece && r.off < lowest) {
               lowest = r.off;
               rv = r;
           }
       }
       if (pendingRequest != null &&
-          pendingRequest.piece == piece && pendingRequest.off < lowest)
+          pendingRequest.getPiece() == piece && pendingRequest.off < lowest)
           rv = pendingRequest;
 
       if (_log.shouldLog(Log.DEBUG))
@@ -450,14 +448,16 @@ class PeerState implements DataLoader
    *  @return List of PartialPieces, even those with an offset == 0, or empty list
    *  @since 0.8.2
    */
-  synchronized List<PartialPiece> returnPartialPieces()
+  synchronized List<Request> returnPartialPieces()
   {
       Set<Integer> pcs = getRequestedPieces();
-      List<PartialPiece> rv = new ArrayList(pcs.size());
+      List<Request> rv = new ArrayList(pcs.size());
       for (Integer p : pcs) {
           Request req = getLowestOutstandingRequest(p.intValue());
-          if (req != null)
-              rv.add(new PartialPiece(req));
+          if (req != null) {
+              req.getPartialPiece().setDownloaded(req.off);
+              rv.add(req);
+          }
       }
       outstandingRequests.clear();
       pendingRequest = null;
@@ -471,9 +471,9 @@ class PeerState implements DataLoader
   synchronized private Set<Integer> getRequestedPieces() {
       Set<Integer> rv = new HashSet(outstandingRequests.size() + 1);
       for (Request req : outstandingRequests) {
-          rv.add(Integer.valueOf(req.piece));
+          rv.add(Integer.valueOf(req.getPiece()));
       if (pendingRequest != null)
-          rv.add(Integer.valueOf(pendingRequest.piece));
+          rv.add(Integer.valueOf(pendingRequest.getPiece()));
       }
       return rv;
   }
@@ -489,6 +489,13 @@ class PeerState implements DataLoader
   /** @since 0.8.2 */
   void extensionMessage(int id, byte[] bs)
   {
+      if (metainfo != null && metainfo.isPrivate() &&
+          (id == ExtensionHandler.ID_METADATA || id == ExtensionHandler.ID_PEX)) {
+          // shouldn't get this since we didn't advertise it but they could send it anyway
+          if (_log.shouldLog(Log.WARN))
+              _log.warn("Private torrent, ignoring ext msg " + id);
+          return;
+      }
       ExtensionHandler.handleMessage(peer, listener, id, bs);
       // Peer coord will get metadata from MagnetState,
       // verify, and then call gotMetaInfo()
@@ -519,10 +526,14 @@ class PeerState implements DataLoader
           setInteresting(true);
   }
 
-  /** @since 0.8.4 */
+  /**
+   *  Unused
+   *  @since 0.8.4
+   */
   void portMessage(int port)
   {
-      listener.gotPort(peer, port);
+      // for compatibility with old DHT PORT message
+      listener.gotPort(peer, port, port + 1);
   }
 
   void unknownMessage(int type, byte[] bs)
@@ -567,19 +578,20 @@ class PeerState implements DataLoader
    * @since 0.8.1
    */
   synchronized void cancelPiece(int piece) {
-        if (lastRequest != null && lastRequest.piece == piece)
+        if (lastRequest != null && lastRequest.getPiece() == piece)
           lastRequest = null;
         
         Iterator<Request> it = outstandingRequests.iterator();
         while (it.hasNext())
           {
             Request req = it.next();
-            if (req.piece == piece)
+            if (req.getPiece() == piece)
               {
                 it.remove();
                 // Send cancel even when we are choked to make sure that it is
                 // really never ever send.
                 out.sendCancel(req);
+                req.getPartialPiece().release();
               }
           }
   }
@@ -590,10 +602,10 @@ class PeerState implements DataLoader
    * @since 0.8.1
    */
   synchronized boolean isRequesting(int piece) {
-      if (pendingRequest != null && pendingRequest.piece == piece)
+      if (pendingRequest != null && pendingRequest.getPiece() == piece)
           return true;
       for (Request req : outstandingRequests) {
-          if (req.piece == piece)
+          if (req.getPiece() == piece)
               return true;
       }
       return false;
@@ -670,12 +682,13 @@ class PeerState implements DataLoader
                   _log.debug(peer + " addRequest() we are choked, delaying requestNextPiece()");
               return;
           }
+          // huh? rv unused
           more_pieces = requestNextPiece();
         } else if (more_pieces) // We want something
           {
             int pieceLength;
             boolean isLastChunk;
-            pieceLength = metainfo.getPieceLength(lastRequest.piece);
+            pieceLength = metainfo.getPieceLength(lastRequest.getPiece());
             isLastChunk = lastRequest.off + lastRequest.len == pieceLength;
 
             // Last part of a piece?
@@ -683,14 +696,13 @@ class PeerState implements DataLoader
               more_pieces = requestNextPiece();
             else
               {
-                    int nextPiece = lastRequest.piece;
+                    PartialPiece nextPiece = lastRequest.getPartialPiece();
                     int nextBegin = lastRequest.off + PARTSIZE;
-                    byte[] bs = lastRequest.bs;
                     int maxLength = pieceLength - nextBegin;
                     int nextLength = maxLength > PARTSIZE ? PARTSIZE
                                                           : maxLength;
                     Request req
-                      = new Request(nextPiece, bs, nextBegin, nextLength);
+                      = new Request(nextPiece,nextBegin, nextLength);
                     outstandingRequests.add(req);
                     if (!choked)
                       out.sendRequest(req);
@@ -700,6 +712,8 @@ class PeerState implements DataLoader
       }
 
     // failsafe
+    // However this is bad as it thrashes the peer when we change our mind
+    // Ticket 691 cause here?
     if (interesting && lastRequest == null && outstandingRequests.isEmpty())
         setInteresting(false);
 
@@ -727,6 +741,10 @@ class PeerState implements DataLoader
                   out.sendRequest(r);
                 lastRequest = r;
                 return true;
+            } else {
+                if (_log.shouldLog(Log.WARN))
+                    _log.warn("Got dup from coord: " + pp);
+                pp.release();
             }
         }
 
@@ -736,7 +754,7 @@ class PeerState implements DataLoader
         // what piece to give us next.
         int nextPiece = listener.wantPiece(peer, bitfield);
         if (nextPiece != -1
-            && (lastRequest == null || lastRequest.piece != nextPiece)) {
+            && (lastRequest == null || lastRequest.getPiece() != nextPiece)) {
                 if (_log.shouldLog(Log.DEBUG))
                     _log.debug(peer + " want piece " + nextPiece);
                 // Fail safe to make sure we are interested
@@ -773,6 +791,8 @@ class PeerState implements DataLoader
     }
 
     // failsafe
+    // However this is bad as it thrashes the peer when we change our mind
+    // Ticket 691 cause here?
     if (outstandingRequests.isEmpty())
         lastRequest = null;
 

apps/i2psnark/java/src/org/klomp/snark/Piece.java

@@ -12,13 +12,13 @@ class Piece implements Comparable {
     private final int id;
     private final Set<PeerID> peers;
     /** @since 0.8.3 */
-    private Set<PeerID> requests;
+    private volatile Set<PeerID> requests;
     /** @since 0.8.1 */
     private int priority;
     
     public Piece(int id) {
         this.id = id;
-        this.peers = new HashSet(I2PSnarkUtil.MAX_CONNECTIONS);
+        this.peers = new HashSet(I2PSnarkUtil.MAX_CONNECTIONS / 2);
         // defer creating requests to save memory
     }
     
@@ -54,9 +54,21 @@ class Piece implements Comparable {
     /** caller must synchronize */
     public boolean addPeer(Peer peer) { return this.peers.add(peer.getPeerID()); }
 
-    /** caller must synchronize */
+    /**
+     * Caller must synchronize.
+     * @return true if removed
+     */
     public boolean removePeer(Peer peer) { return this.peers.remove(peer.getPeerID()); }
 
+    /**
+     * How many peers have this piece?
+     * Caller must synchronize
+     * @since 0.9.1
+     */
+    public int getPeerCount() {
+        return this.peers.size();
+    } 
+    
     /** caller must synchronize */
     public boolean isRequested() {
         return this.requests != null && !this.requests.isEmpty();
@@ -95,6 +107,17 @@ class Piece implements Comparable {
     public int getRequestCount() {
         return this.requests == null ? 0 : this.requests.size();
     } 
+
+    /**
+     * Clear all knowledge of peers
+     * Caller must synchronize
+     * @since 0.9.3
+     */
+    public void clear() {
+        peers.clear();
+        if (requests != null)
+            requests.clear();
+    } 
     
     /** @return default 0 @since 0.8.1 */
     public int getPriority() { return this.priority; }

apps/i2psnark/java/src/org/klomp/snark/Request.java

@@ -20,14 +20,16 @@
 
 package org.klomp.snark;
 
+import java.io.DataInputStream;
+import java.io.IOException;
+
 /**
  * Holds all information needed for a partial piece request.
  * This class should be used only by PeerState, PeerConnectionIn, and PeerConnectionOut.
  */
 class Request
 {
-  final int piece;
-  final byte[] bs;
+  private final PartialPiece piece;
   final int off;
   final int len;
   long sendTime;
@@ -36,26 +38,49 @@ class Request
    * Creates a new Request.
    *
    * @param piece Piece number requested.
-   * @param bs byte array where response should be stored.
    * @param off the offset in the array.
    * @param len the number of bytes requested.
    */
-  Request(int piece, byte[] bs, int off, int len)
+  Request(PartialPiece piece, int off, int len)
   {
+    // Sanity check
+    if (off < 0 || len <= 0 || off + len > piece.getLength())
+      throw new IndexOutOfBoundsException("Illegal Request " + toString());
+
     this.piece = piece;
-    this.bs = bs;
     this.off = off;
     this.len = len;
+  }
 
-    // Sanity check
-    if (piece < 0 || off < 0 || len <= 0 || off + len > bs.length)
-      throw new IndexOutOfBoundsException("Illegal Request " + toString());
+  /**
+   *  @since 0.9.1
+   */
+  public void read(DataInputStream din) throws IOException {
+      piece.read(din, off, len);
+  }
+
+  /**
+   *  The piece number this Request is for
+   *
+   *  @since 0.9.1
+   */
+  public int getPiece() {
+      return piece.getPiece();
+  }
+
+  /**
+   *  The PartialPiece this Request is for
+   *
+   *  @since 0.9.1
+   */
+  public PartialPiece getPartialPiece() {
+      return piece;
   }
 
     @Override
   public int hashCode()
   {
-    return piece ^ off ^ len;
+    return piece.getPiece() ^ off ^ len;
   }
 
     @Override
@@ -64,7 +89,7 @@ class Request
     if (o instanceof Request)
       {
         Request req = (Request)o;
-        return req.piece == piece && req.off == off && req.len == len;
+        return req.piece.equals(piece) && req.off == off && req.len == len;
       }
 
     return false;
@@ -73,6 +98,6 @@ class Request
     @Override
   public String toString()
   {
-    return "(" + piece + "," + off + "," + len + ")";
+    return "(" + piece.getPiece() + "," + off + "," + len + ")";
   }
 }

apps/i2psnark/java/src/org/klomp/snark/Snark.java

@@ -20,12 +20,10 @@
 
 package org.klomp.snark;
 
-import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
@@ -37,6 +35,7 @@ import net.i2p.I2PAppContext;
 import net.i2p.client.streaming.I2PServerSocket;
 import net.i2p.data.Destination;
 import net.i2p.util.I2PThread;
+import net.i2p.util.Log;
 
 /**
  * Main Snark program startup class.
@@ -49,34 +48,12 @@ public class Snark
   private final static int MIN_PORT = 6881;
   private final static int MAX_PORT = 6889;
 
-  // Error messages (non-fatal)
-  public final static int ERROR   = 1;
-
-  // Warning messages
-  public final static int WARNING = 2;
-
-  // Notices (peer level)
-  public final static int NOTICE  = 3;
-
-  // Info messages (protocol policy level)
-  public final static int INFO    = 4;
-
-  // Debug info (protocol level)
-  public final static int DEBUG   = 5;
-
-  // Very low level stuff (network level)
-  public final static int ALL     = 6;
-
-  /**
-   * What level of debug info to show.
-   */
-  //public static int debug = NOTICE;
-
   // Whether or not to ask the user for commands while sharing
-  private static boolean command_interpreter = true;
+  //private static boolean command_interpreter = true;
 
   private static final String newline = System.getProperty("line.separator");
 
+/****
   private static final String copyright =
   "The Hunting of the Snark Project - Copyright (C) 2003 Mark J. Wielaard"
   + newline + newline
@@ -92,10 +69,12 @@ public class Snark
   "Press return for help. Type \"quit\" and return to stop.";
   private static final String help =
   "Commands: 'info', 'list', 'quit'.";
+****/
 
   // String indicating main activity
   String activity = "Not started";
   
+/****
   private static class OOMListener implements I2PThread.OOMEventListener {
       public void outOfMemory(OutOfMemoryError err) {
           try {
@@ -108,6 +87,7 @@ public class Snark
       }
       
   }
+****/
   
 /******** No, not maintaining a command-line client
 
@@ -247,11 +227,13 @@ public class Snark
   private TrackerClient trackerclient;
   private String rootDataDir = ".";
   private final CompleteListener completeListener;
-  private boolean stopped;
+  private volatile boolean stopped;
+  private volatile boolean starting;
   private byte[] id;
-  private byte[] infoHash;
+  private final byte[] infoHash;
   private String additionalTrackerURL;
   private final I2PSnarkUtil _util;
+  private final Log _log;
   private final PeerCoordinatorSet _peerCoordinatorSet;
   private String trackerProblems;
   private int trackerSeenPeers;
@@ -305,6 +287,7 @@ public class Snark
 
     completeListener = complistener;
     _util = util;
+    _log = util.getContext().logManager().getLog(Snark.class);
     _peerCoordinatorSet = peerCoordinatorSet;
     acceptor = connectionAcceptor;
 
@@ -315,10 +298,9 @@ public class Snark
     activity = "Network setup";
 
     id = generateID();
-    debug("My peer id: " + PeerID.idencode(id), Snark.INFO);
+    if (_log.shouldLog(Log.INFO))
+        _log.info("My peer id: " + PeerID.idencode(id));
 
-    int port;
-    IOException lastException = null;
 /*
  * Don't start a tunnel if the torrent isn't going to be started.
  * If we are starting,
@@ -339,6 +321,7 @@ public class Snark
     meta = null;
     File f = null;
     InputStream in = null;
+    byte[] x_infoHash = null;
     try
       {
         f = new File(torrent);
@@ -346,6 +329,8 @@ public class Snark
           in = new FileInputStream(f);
         else
           {
+         /**** No, we don't ever fetch a torrent file this way
+               and we don't want to block in the constructor
             activity = "Getting torrent";
             File torrentFile = _util.get(torrent, 3);
             if (torrentFile == null) {
@@ -355,9 +340,11 @@ public class Snark
                 torrentFile.deleteOnExit();
                 in = new FileInputStream(torrentFile);
             }
+         *****/
+             throw new IOException("not found");
           }
         meta = new MetaInfo(in);
-        infoHash = meta.getInfoHash();
+        x_infoHash = meta.getInfoHash();
       }
     catch(IOException ioe)
       {
@@ -398,7 +385,9 @@ public class Snark
               try { in.close(); } catch (IOException ioe) {}
       }    
 
-    debug(meta.toString(), INFO);
+    infoHash = x_infoHash;  // final
+    if (_log.shouldLog(Log.INFO))
+        _log.info(meta.toString());
     
     // When the metainfo torrent was created from an existing file/dir
     // it already exists.
@@ -459,6 +448,7 @@ public class Snark
   {
     completeListener = complistener;
     _util = util;
+    _log = util.getContext().logManager().getLog(Snark.class);
     _peerCoordinatorSet = peerCoordinatorSet;
     acceptor = connectionAcceptor;
     this.torrent = torrent;
@@ -505,9 +495,19 @@ public class Snark
   }
 
   /**
-   * Start up contacting peers and querying the tracker
+   * Start up contacting peers and querying the tracker.
+   * Blocks if tunnel is not yet open.
    */
-  public void startTorrent() {
+  public synchronized void startTorrent() {
+      starting = true;
+      try {
+          x_startTorrent();
+      } finally {
+          starting = false;
+      }
+  }
+
+  private void x_startTorrent() {
     boolean ok = _util.connect();
     if (!ok) fatal("Unable to connect to I2P");
     if (coordinator == null) {
@@ -516,9 +516,11 @@ public class Snark
             fatal("Unable to listen for I2P connections");
         else {
             Destination d = serversocket.getManager().getSession().getMyDestination();
-            debug("Listening on I2P destination " + d.toBase64() + " / " + d.calculateHash().toBase64(), NOTICE);
+            if (_log.shouldLog(Log.INFO))
+                _log.info("Listening on I2P destination " + d.toBase64() + " / " + d.calculateHash().toBase64());
         }
-        debug("Starting PeerCoordinator, ConnectionAcceptor, and TrackerClient", NOTICE);
+        if (_log.shouldLog(Log.INFO))
+            _log.info("Starting PeerCoordinator, ConnectionAcceptor, and TrackerClient");
         activity = "Collecting pieces";
         coordinator = new PeerCoordinator(_util, id, infoHash, meta, storage, this, this);
         if (_peerCoordinatorSet != null) {
@@ -538,21 +540,14 @@ public class Snark
     }
 
     stopped = false;
-    boolean coordinatorChanged = false;
     if (coordinator.halted()) {
-        // ok, we have already started and stopped, but the coordinator seems a bit annoying to
-        // restart safely, so lets build a new one to replace the old
+        coordinator.restart();
         if (_peerCoordinatorSet != null)
-            _peerCoordinatorSet.remove(coordinator);
-        PeerCoordinator newCoord = new PeerCoordinator(_util, id, infoHash, meta, storage, this, this);
-        if (_peerCoordinatorSet != null)
-            _peerCoordinatorSet.add(newCoord);
-        coordinator = newCoord;
-        coordinatorChanged = true;
+            _peerCoordinatorSet.add(coordinator);
     }
-    if (!trackerclient.started() && !coordinatorChanged) {
+    if (!trackerclient.started()) {
         trackerclient.start();
-    } else if (trackerclient.halted() || coordinatorChanged) {
+    } else if (trackerclient.halted()) {
         if (storage != null) {
             try {
                  storage.reopen(rootDataDir);
@@ -563,23 +558,30 @@ public class Snark
                  fatal("Could not reopen storage", ioe);
              }
         }
-        TrackerClient newClient = new TrackerClient(_util, meta, additionalTrackerURL, coordinator, this);
-        if (!trackerclient.halted())
-            trackerclient.halt();
-        trackerclient = newClient;
         trackerclient.start();
     } else {
-        debug("NOT starting TrackerClient???", NOTICE);
+        if (_log.shouldLog(Log.INFO))
+            _log.info("NOT starting TrackerClient???");
     }
   }
+
   /**
    * Stop contacting the tracker and talking with peers
    */
   public void stopTorrent() {
+      stopTorrent(false);
+  }
+
+  /**
+   * Stop contacting the tracker and talking with peers
+   * @param fast if true, limit the life of the unannounce threads
+   * @since 0.9.1
+   */
+  public synchronized void stopTorrent(boolean fast) {
     stopped = true;
     TrackerClient tc = trackerclient;
     if (tc != null)
-        tc.halt();
+        tc.halt(fast);
     PeerCoordinator pc = coordinator;
     if (pc != null)
         pc.halt();
@@ -601,10 +603,12 @@ public class Snark
         _util.disconnect();
   }
 
+/****
   private static Snark parseArguments(String[] args)
   {
     return parseArguments(args, null, null);
   }
+****/
 
     // Accessors
 
@@ -668,6 +672,38 @@ public class Snark
         return stopped;
     }
 
+    /**
+     *  Startup in progress.
+     *  @since 0.9.1
+     */
+    public boolean isStarting() {
+        return starting && stopped;
+    }
+
+    /**
+     *  Set startup in progress.
+     *  @since 0.9.1
+     */
+    public void setStarting() {
+        starting = true;
+    }
+
+    /**
+     *  File checking in progress.
+     *  @since 0.9.3
+     */
+    public boolean isChecking() {
+        return storage != null && storage.isChecking();
+    }
+
+    /**
+     *  Disk allocation (ballooning) in progress.
+     *  @since 0.9.3
+     */
+    public boolean isAllocating() {
+        return storage != null && storage.isAllocating();
+    }
+
     /**
      *  @since 0.8.4
      */
@@ -780,6 +816,7 @@ public class Snark
     }
 
     /**
+     *  Bytes not yet in storage. Does NOT account for skipped files.
      *  @return exact value. or -1 if no storage yet.
      *          getNeeded() * pieceLength(0) isn't accurate if last piece
      *          is still needed.
@@ -800,6 +837,20 @@ public class Snark
     }
 
     /**
+     *  Bytes still wanted. DOES account for skipped files.
+     *  FIXME -1 when not running.
+     *  @return exact value. or -1 if no storage yet or when not running.
+     *  @since 0.9.1
+     */
+    public long getNeededLength() {
+        PeerCoordinator coord = coordinator;
+        if (coord != null)
+            return coord.getNeededLength();
+        return -1;
+    }
+
+    /**
+     *  Does not account for skipped files.
      *  @return number of pieces still needed (magnet mode or not), or -1 if unknown
      *  @since 0.8.4
      */
@@ -911,7 +962,7 @@ public class Snark
           }
         else if (args[i].equals("--no-commands"))
           {
-            command_interpreter = false;
+            //command_interpreter = false;
             i++;
           }
         //else if (args[i].equals("--eepproxy"))
@@ -970,22 +1021,13 @@ public class Snark
   private static void usage()
   {
     System.out.println
-      ("Usage: snark [--debug [level]] [--no-commands] [--port <port>]");
+      ("Usage: snark [--no-commands] [--port <port>]");
     System.out.println
       ("             [--eepproxy hostname portnum]");
     System.out.println
       ("             [--i2cp routerHost routerPort ['name=val name=val name=val']]");
     System.out.println
       ("             (<url>|<file>)");
-    System.out.println
-      ("  --debug\tShows some extra info and stacktraces");
-    System.out.println
-      ("    level\tHow much debug details to show");
-    System.out.println
-      ("         \t(defaults to "
-       + NOTICE + ", with --debug to "
-       + INFO + ", highest level is "
-       + ALL + ").");
     System.out.println
       ("  --no-commands\tDon't read interactive commands or show usage info.");
     System.out.println
@@ -1024,22 +1066,18 @@ public class Snark
    */
   private void fatal(String s, Throwable t)
   {
-    _util.debug(s, ERROR, t);
+    _log.error(s, t);
     //System.err.println("snark: " + s + ((t == null) ? "" : (": " + t)));
     //if (debug >= INFO && t != null)
     //  t.printStackTrace();
     stopTorrent();
+    if (t != null)
+        s += ": " + t;
+    if (completeListener != null)
+        completeListener.fatal(this, s);
     throw new RuntimeException(s, t);
   }
 
-  /**
-   * Show debug info if debug is true.
-   */
-  private void debug(String s, int level)
-  {
-    _util.debug(s, level, null);
-  }
-
   /** CoordinatorListener - this does nothing */
   public void peerChange(PeerCoordinator coordinator, Peer peer)
   {
@@ -1056,10 +1094,12 @@ public class Snark
    * @since 0.8.4
    */
   public void gotMetaInfo(PeerCoordinator coordinator, MetaInfo metainfo) {
-      meta = metainfo;
       try {
-          storage = new Storage(_util, meta, this);
+          // The following two may throw IOE...
+          storage = new Storage(_util, metainfo, this);
           storage.check(rootDataDir);
+          // ... so don't set meta until here
+          meta = metainfo;
           if (completeListener != null) {
               String newName = completeListener.gotMetaInfo(this);
               if (newName != null)
@@ -1075,7 +1115,12 @@ public class Snark
       }
   }
 
-  private boolean allocating = false;
+
+  ///////////// Begin StorageListener methods
+
+  //private boolean allocating = false;
+
+  /** does nothing */
   public void storageCreateFile(Storage storage, String name, long length)
   {
     //if (allocating)
@@ -1083,27 +1128,29 @@ public class Snark
 
     //System.out.print("Creating file '" + name
     //                 + "' of length " + length + ": ");
-    allocating = true;
+    //allocating = true;
   }
 
   // How much storage space has been allocated
   private long allocated = 0;
 
+  /** does nothing */
   public void storageAllocated(Storage storage, long length)
   {
-    allocating = true;
+    //allocating = true;
     //System.out.print(".");
-    allocated += length;
+    //allocated += length;
     //if (allocated == meta.getTotalLength())
     //  System.out.println(); // We have all the disk space we need.
   }
 
   private boolean allChecked = false;
   private boolean checking = false;
-  private boolean prechecking = true;
+  //private boolean prechecking = true;
+
   public void storageChecked(Storage storage, int num, boolean checked)
   {
-    allocating = false;
+    //allocating = false;
     if (!allChecked && !checking)
       {
         // Use the MetaInfo from the storage since our own might not
@@ -1115,9 +1162,12 @@ public class Snark
         //                   + " pieces: ");
         checking = true;
       }
-    if (!checking)
-      debug("Got " + (checked ? "" : "BAD ") + "piece: " + num,
-                  Snark.INFO);
+    if (!checking) {
+        if (_log.shouldLog(Log.INFO))
+            _log.info("Got " + (checked ? "" : "BAD ") + "piece: " + num);
+        if (completeListener != null)
+            completeListener.gotPiece(this);
+    }
   }
 
   public void storageAllChecked(Storage storage)
@@ -1133,7 +1183,8 @@ public class Snark
   
   public void storageCompleted(Storage storage)
   {
-    debug("Completely received " + torrent, Snark.INFO);
+    if (_log.shouldLog(Log.INFO))
+        _log.info("Completely received " + torrent);
     //storage.close();
     //System.out.println("Completely received: " + torrent);
     if (completeListener != null)
@@ -1145,6 +1196,9 @@ public class Snark
     coordinator.setWantedPieces();
   }
 
+  ///////////// End StorageListener methods
+
+
   /** SnarkSnutdown callback unused */
   public void shutdown()
   {
@@ -1153,25 +1207,16 @@ public class Snark
     //System.exit(0);
   }
   
-  public interface CompleteListener {
-    public void torrentComplete(Snark snark);
-    public void updateStatus(Snark snark);
-
-    /**
-     * We transitioned from magnet mode, we have now initialized our
-     * metainfo and storage. The listener should now call getMetaInfo()
-     * and save the data to disk.
-     *
-     * @return the new name for the torrent or null on error
-     * @since 0.8.4
-     */
-    public String gotMetaInfo(Snark snark);
-
-    // not really listeners but the easiest way to get back to an optional SnarkManager
-    public long getSavedTorrentTime(Snark snark);
-    public BitField getSavedTorrentBitField(Snark snark);
+  /**
+   * StorageListener and CoordinatorListener callback
+   * @since 0.9.2
+   */
+  public void addMessage(String message) {
+      if (completeListener != null)
+          completeListener.addMessage(this, message);
   }
 
+
   /** Maintain a configurable total uploader cap
    * coordinatorListener
    */
@@ -1181,8 +1226,8 @@ public class Snark
     if (_peerCoordinatorSet == null || uploaders <= 0)
       return false;
     int totalUploaders = 0;
-    for (Iterator iter = _peerCoordinatorSet.iterator(); iter.hasNext(); ) {
-      PeerCoordinator c = (PeerCoordinator)iter.next();
+    for (Iterator<PeerCoordinator> iter = _peerCoordinatorSet.iterator(); iter.hasNext(); ) {
+      PeerCoordinator c = iter.next();
       if (!c.halted())
         totalUploaders += c.uploaders;
     }
@@ -1195,13 +1240,14 @@ public class Snark
     if (_peerCoordinatorSet == null)
       return false;
     long total = 0;
-    for (Iterator iter = _peerCoordinatorSet.iterator(); iter.hasNext(); ) {
-      PeerCoordinator c = (PeerCoordinator)iter.next();
+    for (Iterator<PeerCoordinator> iter = _peerCoordinatorSet.iterator(); iter.hasNext(); ) {
+      PeerCoordinator c = iter.next();
       if (!c.halted())
         total += c.getCurrentUploadRate();
     }
     long limit = 1024l * _util.getMaxUpBW();
-    debug("Total up bw: " + total + " Limit: " + limit, Snark.WARNING);
+    if (_log.shouldLog(Log.INFO))
+        _log.info("Total up bw: " + total + " Limit: " + limit);
     return total > limit;
   }
 

apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java

@@ -8,6 +8,10 @@ import java.io.FilenameFilter;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -16,13 +20,14 @@ import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
 import java.util.StringTokenizer;
-import java.util.TreeMap;
-import java.util.Collection;
+import java.util.Queue;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.LinkedBlockingQueue;
 
 import net.i2p.I2PAppContext;
 import net.i2p.data.Base64;
 import net.i2p.data.DataHelper;
+import net.i2p.update.*;
 import net.i2p.util.ConcurrentHashSet;
 import net.i2p.util.FileUtil;
 import net.i2p.util.I2PAppThread;
@@ -30,13 +35,15 @@ import net.i2p.util.Log;
 import net.i2p.util.OrderedProperties;
 import net.i2p.util.SecureDirectory;
 import net.i2p.util.SecureFileOutputStream;
+import net.i2p.util.SimpleScheduler;
+import net.i2p.util.SimpleTimer;
+
+import org.klomp.snark.dht.DHT;
 
 /**
  * Manage multiple snarks
  */
-public class SnarkManager implements Snark.CompleteListener {
-    private static SnarkManager _instance = new SnarkManager();
-    public static SnarkManager instance() { return _instance; }
+public class SnarkManager implements CompleteListener {
     
     /**
      *  Map of (canonical) filename of the .torrent file to Snark instance.
@@ -51,12 +58,16 @@ public class SnarkManager implements Snark.CompleteListener {
     private Properties _config;
     private final I2PAppContext _context;
     private final Log _log;
-    private final List _messages;
+    private final Queue<String> _messages;
     private final I2PSnarkUtil _util;
     private PeerCoordinatorSet _peerCoordinatorSet;
     private ConnectionAcceptor _connectionAcceptor;
     private Thread _monitor;
     private volatile boolean _running;
+    private volatile boolean _stopping;
+    private final Map<String, Tracker> _trackerMap;
+    private UpdateManager _umgr;
+    private UpdateHandler _uhandler;
     
     public static final String PROP_I2CP_HOST = "i2psnark.i2cpHost";
     public static final String PROP_I2CP_PORT = "i2psnark.i2cpPort";
@@ -79,27 +90,56 @@ public class SnarkManager implements Snark.CompleteListener {
     //public static final String DEFAULT_LINK_PREFIX = "file:///";
     public static final String PROP_STARTUP_DELAY = "i2psnark.startupDelay";
     public static final String PROP_REFRESH_DELAY = "i2psnark.refreshSeconds";
+    public static final String RC_PROP_THEME = "routerconsole.theme";
+    public static final String RC_PROP_UNIVERSAL_THEMING = "routerconsole.universal.theme";
     public static final String PROP_THEME = "i2psnark.theme";
     public static final String DEFAULT_THEME = "ubergine";
     private static final String PROP_USE_OPENTRACKERS = "i2psnark.useOpentrackers";
     public static final String PROP_OPENTRACKERS = "i2psnark.opentrackers";
+    public static final String PROP_PRIVATETRACKERS = "i2psnark.privatetrackers";
+    private static final String PROP_USE_DHT = "i2psnark.enableDHT";
 
     public static final int MIN_UP_BW = 2;
     public static final int DEFAULT_MAX_UP_BW = 10;
     public static final int DEFAULT_STARTUP_DELAY = 3; 
     public static final int DEFAULT_REFRESH_DELAY_SECS = 60;
 
-    private SnarkManager() {
+    /**
+     *  "name", "announceURL=websiteURL" pairs
+     *  '=' in announceURL must be escaped as &#44;
+     */
+    public static final String DEFAULT_TRACKERS[] = { 
+//       "Postman", "http://YRgrgTLGnbTq2aZOZDJQ~o6Uk5k6TK-OZtx0St9pb0G-5EGYURZioxqYG8AQt~LgyyI~NCj6aYWpPO-150RcEvsfgXLR~CxkkZcVpgt6pns8SRc3Bi-QSAkXpJtloapRGcQfzTtwllokbdC-aMGpeDOjYLd8b5V9Im8wdCHYy7LRFxhEtGb~RL55DA8aYOgEXcTpr6RPPywbV~Qf3q5UK55el6Kex-6VCxreUnPEe4hmTAbqZNR7Fm0hpCiHKGoToRcygafpFqDw5frLXToYiqs9d4liyVB-BcOb0ihORbo0nS3CLmAwZGvdAP8BZ7cIYE3Z9IU9D1G8JCMxWarfKX1pix~6pIA-sp1gKlL1HhYhPMxwyxvuSqx34o3BqU7vdTYwWiLpGM~zU1~j9rHL7x60pVuYaXcFQDR4-QVy26b6Pt6BlAZoFmHhPcAuWfu-SFhjyZYsqzmEmHeYdAwa~HojSbofg0TMUgESRXMw6YThK1KXWeeJVeztGTz25sL8AAAA.i2p/announce.php=http://tracker.postman.i2p/"
+//       , "eBook", "http://E71FRom6PZNEqTN2Lr8P-sr23b7HJVC32KoGnVQjaX6zJiXwhJy2HsXob36Qmj81TYFZdewFZa9mSJ533UZgGyQkXo2ahctg82JKYZfDe5uDxAn1E9YPjxZCWJaFJh0S~UwSs~9AZ7UcauSJIoNtpxrtbmRNVFLqnkEDdLZi26TeucfOmiFmIWnVblLniWv3tG1boE9Abd-6j3FmYVrRucYuepAILYt6katmVNOk6sXmno1Eynrp~~MBuFq0Ko6~jsc2E2CRVYXDhGHEMdt-j6JUz5D7S2RIVzDRqQyAZLKJ7OdQDmI31przzmne1vOqqqLC~1xUumZVIvF~yOeJUGNjJ1Vx0J8i2BQIusn1pQJ6UCB~ZtZZLQtEb8EPVCfpeRi2ri1M5CyOuxN0V5ekmPHrYIBNevuTCRC26NP7ZS5VDgx1~NaC3A-CzJAE6f1QXi0wMI9aywNG5KGzOPifcsih8eyGyytvgLtrZtV7ykzYpPCS-rDfITncpn5hliPUAAAA.i2p/pub/bt/announce.php=http://de-ebook-archiv.i2p/pub/bt/"
+//       , "Gaytorrents", "http://uxPWHbK1OIj9HxquaXuhMiIvi21iK0~ZiG9d8G0840ZXIg0r6CbiV71xlsqmdnU6wm0T2LySriM0doW2gUigo-5BNkUquHwOjLROiETnB3ZR0Ml4IGa6QBPn1aAq2d9~g1r1nVjLE~pcFnXB~cNNS7kIhX1d6nLgYVZf0C2cZopEow2iWVUggGGnAA9mHjE86zLEnTvAyhbAMTqDQJhEuLa0ZYSORqzJDMkQt90MV4YMjX1ICY6RfUSFmxEqu0yWTrkHsTtRw48l~dz9wpIgc0a0T9C~eeWvmBFTqlJPtQZwntpNeH~jF7nlYzB58olgV2HHFYpVYD87DYNzTnmNWxCJ5AfDorm6AIUCV2qaE7tZtI1h6fbmGpGlPyW~Kw5GXrRfJwNvr6ajwAVi~bPVnrBwDZezHkfW4slOO8FACPR28EQvaTu9nwhAbqESxV2hCTq6vQSGjuxHeOuzBOEvRWkLKOHWTC09t2DbJ94FSqETmZopTB1ukEmaxRWbKSIaAAAA.i2p/announce.php=http://gaytorrents.i2p/"
+//       , "NickyB", "http://9On6d3cZ27JjwYCtyJJbowe054d5tFnfMjv4PHsYs-EQn4Y4mk2zRixatvuAyXz2MmRfXG-NAUfhKr0KCxRNZbvHmlckYfT-WBzwwpiMAl0wDFY~Pl8cqXuhfikSG5WrqdPfDNNIBuuznS0dqaczf~OyVaoEOpvuP3qV6wKqbSSLpjOwwAaQPHjlRtNIW8-EtUZp-I0LT45HSoowp~6b7zYmpIyoATvIP~sT0g0MTrczWhbVTUZnEkZeLhOR0Duw1-IRXI2KHPbA24wLO9LdpKKUXed05RTz0QklW5ROgR6TYv7aXFufX8kC0-DaKvQ5JKG~h8lcoHvm1RCzNqVE-2aiZnO2xH08H-iCWoLNJE-Td2kT-Tsc~3QdQcnEUcL5BF-VT~QYRld2--9r0gfGl-yDrJZrlrihHGr5J7ImahelNn9PpkVp6eIyABRmJHf2iicrk3CtjeG1j9OgTSwaNmEpUpn4aN7Kx0zNLdH7z6uTgCGD9Kmh1MFYrsoNlTp4AAAA.i2p/bittorrent/announce.php=http://nickyb.i2p/bittorrent/"
+//       , "Orion", "http://gKik1lMlRmuroXVGTZ~7v4Vez3L3ZSpddrGZBrxVriosCQf7iHu6CIk8t15BKsj~P0JJpxrofeuxtm7SCUAJEr0AIYSYw8XOmp35UfcRPQWyb1LsxUkMT4WqxAT3s1ClIICWlBu5An~q-Mm0VFlrYLIPBWlUFnfPR7jZ9uP5ZMSzTKSMYUWao3ejiykr~mtEmyls6g-ZbgKZawa9II4zjOy-hdxHgP-eXMDseFsrym4Gpxvy~3Fv9TuiSqhpgm~UeTo5YBfxn6~TahKtE~~sdCiSydqmKBhxAQ7uT9lda7xt96SS09OYMsIWxLeQUWhns-C~FjJPp1D~IuTrUpAFcVEGVL-BRMmdWbfOJEcWPZ~CBCQSO~VkuN1ebvIOr9JBerFMZSxZtFl8JwcrjCIBxeKPBmfh~xYh16BJm1BBBmN1fp2DKmZ2jBNkAmnUbjQOqWvUcehrykWk5lZbE7bjJMDFH48v3SXwRuDBiHZmSbsTY6zhGY~GkMQHNGxPMMSIAAAA.i2p/bt/announce.php=http://orion.i2p/bt/"
+//       , "anonymity", "http://8EoJZIKrWgGuDrxA3nRJs1jsPfiGwmFWL91hBrf0HA7oKhEvAna4Ocx47VLUR9retVEYBAyWFK-eZTPcvhnz9XffBEiJQQ~kFSCqb1fV6IfPiV3HySqi9U5Caf6~hC46fRd~vYnxmaBLICT3N160cxBETqH3v2rdxdJpvYt8q4nMk9LUeVXq7zqCTFLLG5ig1uKgNzBGe58iNcsvTEYlnbYcE930ABmrzj8G1qQSgSwJ6wx3tUQNl1z~4wSOUMan~raZQD60lRK70GISjoX0-D0Po9WmPveN3ES3g72TIET3zc3WPdK2~lgmKGIs8GgNLES1cXTolvbPhdZK1gxddRMbJl6Y6IPFyQ9o4-6Rt3Lp-RMRWZ2TG7j2OMcNSiOmATUhKEFBDfv-~SODDyopGBmfeLw16F4NnYednvn4qP10dyMHcUASU6Zag4mfc2-WivrOqeWhD16fVAh8MoDpIIT~0r9XmwdaVFyLcjbXObabJczxCAW3fodQUnvuSkwzAAAA.i2p/anonymityTracker/announce.php=http://anonymityweb.i2p/anonymityTracker/"
+//       , "The freak's tracker", "http://mHKva9x24E5Ygfey2llR1KyQHv5f8hhMpDMwJDg1U-hABpJ2NrQJd6azirdfaR0OKt4jDlmP2o4Qx0H598~AteyD~RJU~xcWYdcOE0dmJ2e9Y8-HY51ie0B1yD9FtIV72ZI-V3TzFDcs6nkdX9b81DwrAwwFzx0EfNvK1GLVWl59Ow85muoRTBA1q8SsZImxdyZ-TApTVlMYIQbdI4iQRwU9OmmtefrCe~ZOf4UBS9-KvNIqUL0XeBSqm0OU1jq-D10Ykg6KfqvuPnBYT1BYHFDQJXW5DdPKwcaQE4MtAdSGmj1epDoaEBUa9btQlFsM2l9Cyn1hzxqNWXELmx8dRlomQLlV4b586dRzW~fLlOPIGC13ntPXogvYvHVyEyptXkv890jC7DZNHyxZd5cyrKC36r9huKvhQAmNABT2Y~pOGwVrb~RpPwT0tBuPZ3lHYhBFYmD8y~AOhhNHKMLzea1rfwTvovBMByDdFps54gMN1mX4MbCGT4w70vIopS9yAAAA.i2p/bytemonsoon/announce.php"
+//       , "mastertracker", "http://VzXD~stRKbL3MOmeTn1iaCQ0CFyTmuFHiKYyo0Rd~dFPZFCYH-22rT8JD7i-C2xzYFa4jT5U2aqHzHI-Jre4HL3Ri5hFtZrLk2ax3ji7Qfb6qPnuYkuiF2E2UDmKUOppI8d9Ye7tjdhQVCy0izn55tBaB-U7UWdcvSK2i85sauyw3G0Gfads1Rvy5-CAe2paqyYATcDmGjpUNLoxbfv9KH1KmwRTNH6k1v4PyWYYnhbT39WfKMbBjSxVQRdi19cyJrULSWhjxaQfJHeWx5Z8Ev4bSPByBeQBFl2~4vqy0S5RypINsRSa3MZdbiAAyn5tr5slWR6QdoqY3qBQgBJFZppy-3iWkFqqKgSxCPundF8gdDLC5ddizl~KYcYKl42y9SGFHIukH-TZs8~em0~iahzsqWVRks3zRG~tlBcX2U3M2~OJs~C33-NKhyfZT7-XFBREvb8Szmd~p66jDxrwOnKaku-G6DyoQipJqIz4VHmY9-y5T8RrUcJcM-5lVoMpAAAA.i2p/announce.php=http://tracker.mastertracker.i2p/"
+//       , "Galen", "http://5jpwQMI5FT303YwKa5Rd38PYSX04pbIKgTaKQsWbqoWjIfoancFdWCShXHLI5G5ofOb0Xu11vl2VEMyPsg1jUFYSVnu4-VfMe3y4TKTR6DTpetWrnmEK6m2UXh91J5DZJAKlgmO7UdsFlBkQfR2rY853-DfbJtQIFl91tbsmjcA5CGQi4VxMFyIkBzv-pCsuLQiZqOwWasTlnzey8GcDAPG1LDcvfflGV~6F5no9mnuisZPteZKlrv~~TDoXTj74QjByWc4EOYlwqK8sbU9aOvz~s31XzErbPTfwiawiaZ0RUI-IDrKgyvmj0neuFTWgjRGVTH8bz7cBZIc3viy6ioD-eMQOrXaQL0TCWZUelRwHRvgdPiQrxdYQs7ixkajeHzxi-Pq0EMm5Vbh3j3Q9kfUFW3JjFDA-MLB4g6XnjCbM5J1rC0oOBDCIEfhQkszru5cyLjHiZ5yeA0VThgu~c7xKHybv~OMXION7V8pBKOgET7ZgAkw1xgYe3Kkyq5syAAAA.i2p/tr/announce.php=http://galen.i2p/tr/"
+       "Postman", "http://tracker2.postman.i2p/announce.php=http://tracker2.postman.i2p/"
+       ,"Welterde", "http://tracker.welterde.i2p/a=http://tracker.welterde.i2p/stats?mode=top5"
+       ,"Diftracker", "http://diftracker.i2p/announce.php=http://diftracker.i2p/"
+//       , "CRSTRACK", "http://b4G9sCdtfvccMAXh~SaZrPqVQNyGQbhbYMbw6supq2XGzbjU4NcOmjFI0vxQ8w1L05twmkOvg5QERcX6Mi8NQrWnR0stLExu2LucUXg1aYjnggxIR8TIOGygZVIMV3STKH4UQXD--wz0BUrqaLxPhrm2Eh9Hwc8TdB6Na4ShQUq5Xm8D4elzNUVdpM~RtChEyJWuQvoGAHY3ppX-EJJLkiSr1t77neS4Lc-KofMVmgI9a2tSSpNAagBiNI6Ak9L1T0F9uxeDfEG9bBSQPNMOSUbAoEcNxtt7xOW~cNOAyMyGydwPMnrQ5kIYPY8Pd3XudEko970vE0D6gO19yoBMJpKx6Dh50DGgybLQ9CpRaynh2zPULTHxm8rneOGRcQo8D3mE7FQ92m54~SvfjXjD2TwAVGI~ae~n9HDxt8uxOecAAvjjJ3TD4XM63Q9TmB38RmGNzNLDBQMEmJFpqQU8YeuhnS54IVdUoVQFqui5SfDeLXlSkh4vYoMU66pvBfWbAAAA.i2p/tracker/announce.php=http://crstrack.i2p/tracker/"
+//       ,"Exotrack", "http://blbgywsjubw3d2zih2giokakhe3o2cko7jtte4risb3hohbcoyva.b32.i2p/announce.php=http://exotrack.i2p/"
+    };
+    
+    /** comma delimited list of name=announceURL=baseURL for the trackers to be displayed */
+    public static final String PROP_TRACKERS = "i2psnark.trackers";
+
+    public SnarkManager(I2PAppContext ctx) {
         _snarks = new ConcurrentHashMap();
         _magnets = new ConcurrentHashSet();
         _addSnarkLock = new Object();
-        _context = I2PAppContext.getGlobalContext();
+        _context = ctx;
         _log = _context.logManager().getLog(SnarkManager.class);
-        _messages = new ArrayList(16);
+        _messages = new LinkedBlockingQueue();
         _util = new I2PSnarkUtil(_context);
         _configFile = new File(CONFIG_FILE);
         if (!_configFile.isAbsolute())
             _configFile = new File(_context.getConfigDir(), CONFIG_FILE);
+        _trackerMap = new ConcurrentHashMap(4);
         loadConfig(null);
     }
 
@@ -112,35 +152,72 @@ public class SnarkManager implements Snark.CompleteListener {
         _connectionAcceptor = new ConnectionAcceptor(_util);
         _monitor = new I2PAppThread(new DirMonitor(), "Snark DirMonitor", true);
         _monitor.start();
-        _context.addShutdownTask(new SnarkManagerShutdown());
+        // delay until UpdateManager is there
+        _context.simpleScheduler().addEvent(new Register(), 4*60*1000);
+        // Not required, Jetty has a shutdown hook
+        //_context.addShutdownTask(new SnarkManagerShutdown());
     }
 
+    /** @since 0.9.4 */
+    private class Register implements SimpleTimer.TimedEvent {
+        public void timeReached() {
+            if (!_running)
+                return;
+            _umgr = _context.updateManager();
+            if (_umgr != null) {
+                _uhandler = new UpdateHandler(_context, _umgr, SnarkManager.this);
+                _umgr.register(_uhandler, UpdateType.ROUTER_SIGNED, UpdateMethod.TORRENT, 10);
+                _log.warn("Registering with update manager");
+            } else {
+                _log.warn("No update manager to register with");
+            }
+        }
+    }
+    
+    /*
+     *  Called by the webapp at Jetty shutdown.
+     *  Stops all torrents. Does not close the tunnel, so the announces have a chance.
+     *  Fix this so an individual webaapp stop will close the tunnel.
+     *  Runs inline.
+     */
     public void stop() {
+        if (_umgr != null && _uhandler != null) {
+            //_uhandler.shutdown();
+            _umgr.unregister(_uhandler, UpdateType.ROUTER_SIGNED, UpdateMethod.TORRENT);
+        }
         _running = false;
         _monitor.interrupt();
         _connectionAcceptor.halt();
-        (new SnarkManagerShutdown()).run();
+        stopAllTorrents(true);
     }
     
+    /** @since 0.9.1 */
+    public boolean isStopping() { return _stopping; }
+
     /** hook to I2PSnarkUtil for the servlet */
     public I2PSnarkUtil util() { return _util; }
 
-    private static final int MAX_MESSAGES = 5;
+    private static final int MAX_MESSAGES = 100;
+
     public void addMessage(String message) {
-        synchronized (_messages) {
-            _messages.add(message);
-            while (_messages.size() > MAX_MESSAGES)
-                _messages.remove(0);
+        _messages.offer(message);
+        while (_messages.size() > MAX_MESSAGES) {
+            _messages.poll();
         }
         if (_log.shouldLog(Log.INFO))
             _log.info("MSG: " + message);
     }
     
     /** newest last */
-    public List getMessages() {
-        synchronized (_messages) {
-            return new ArrayList(_messages);
-        }
+    public List<String> getMessages() {
+        if (_messages.isEmpty())
+            return Collections.EMPTY_LIST;
+        return new ArrayList(_messages);
+    }
+    
+    /** @since 0.9 */
+    public void clearMessages() {
+            _messages.clear();
     }
     
     /**
@@ -148,11 +225,11 @@ public class SnarkManager implements Snark.CompleteListener {
      *  @since 0.8.9
      */
     public boolean areFilesPublic() {
-        return Boolean.valueOf(_config.getProperty(PROP_FILES_PUBLIC)).booleanValue();
+        return Boolean.parseBoolean(_config.getProperty(PROP_FILES_PUBLIC));
     }
 
     public boolean shouldAutoStart() {
-        return Boolean.valueOf(_config.getProperty(PROP_AUTO_START, DEFAULT_AUTO_START)).booleanValue();
+        return Boolean.parseBoolean(_config.getProperty(PROP_AUTO_START, DEFAULT_AUTO_START));
     }
 
 /****
@@ -237,6 +314,9 @@ public class SnarkManager implements Snark.CompleteListener {
             _config.setProperty(PROP_STARTUP_DELAY, Integer.toString(DEFAULT_STARTUP_DELAY));
         if (!_config.containsKey(PROP_THEME))
             _config.setProperty(PROP_THEME, DEFAULT_THEME);
+        // no, so we can switch default to true later
+        //if (!_config.containsKey(PROP_USE_DHT))
+        //    _config.setProperty(PROP_USE_DHT, Boolean.toString(I2PSnarkUtil.DEFAULT_USE_DHT));
         updateConfig();
     }
     /**
@@ -245,6 +325,26 @@ public class SnarkManager implements Snark.CompleteListener {
      */
     public String getTheme() {
         String theme = _config.getProperty(PROP_THEME);
+        boolean universalTheming = _context.getBooleanProperty(RC_PROP_UNIVERSAL_THEMING);
+        if (universalTheming) {
+            // Fetch routerconsole theme (or use our default if it doesn't exist)
+            theme = _context.getProperty(RC_PROP_THEME, DEFAULT_THEME);
+            // Ensure that theme exists
+            String[] themes = getThemes();
+            boolean themeExists = false;
+            for (int i = 0; i < themes.length; i++) {
+                if (themes[i].equals(theme))
+                    themeExists = true;
+            }
+            if (!themeExists) {
+                // Since the default is not "light", explicitly check if universal theme is "classic"
+                if (theme.equals("classic"))
+                    theme = "light";
+                else
+                    theme = DEFAULT_THEME;
+                _config.setProperty(PROP_THEME, DEFAULT_THEME);
+            }
+        }
         return theme;
     }
 
@@ -307,11 +407,15 @@ public class SnarkManager implements Snark.CompleteListener {
         _util.setFilesPublic(areFilesPublic());
         String ot = _config.getProperty(PROP_OPENTRACKERS);
         if (ot != null)
-            _util.setOpenTrackerString(ot);
+            _util.setOpenTrackers(getOpenTrackers());
         String useOT = _config.getProperty(PROP_USE_OPENTRACKERS);
-        boolean bOT = useOT == null || Boolean.valueOf(useOT).booleanValue();
+        boolean bOT = useOT == null || Boolean.parseBoolean(useOT);
         _util.setUseOpenTrackers(bOT);
+        // careful, so we can switch default to true later
+        _util.setUseDHT(Boolean.parseBoolean(_config.getProperty(PROP_USE_DHT,
+                                          Boolean.toString(I2PSnarkUtil.DEFAULT_USE_DHT))));
         getDataDir().mkdirs();
+        initTrackerMap();
     }
     
     private int getInt(String prop, int defaultVal) {
@@ -328,7 +432,7 @@ public class SnarkManager implements Snark.CompleteListener {
     public void updateConfig(String dataDir, boolean filesPublic, boolean autoStart, String refreshDelay,
                              String startDelay, String seedPct, String eepHost, 
                              String eepPort, String i2cpHost, String i2cpPort, String i2cpOpts,
-                             String upLimit, String upBW, boolean useOpenTrackers, String openTrackers, String theme) {
+                             String upLimit, String upBW, boolean useOpenTrackers, boolean useDHT, String theme) {
         boolean changed = false;
         //if (eepHost != null) {
         //    // unused, we use socket eepget
@@ -512,13 +616,16 @@ public class SnarkManager implements Snark.CompleteListener {
             _util.setUseOpenTrackers(useOpenTrackers);
             changed = true;
         }
-        if (openTrackers != null) {
-            if (openTrackers.trim().length() > 0 && !openTrackers.trim().equals(_util.getOpenTrackerString())) {
-                _config.setProperty(PROP_OPENTRACKERS, openTrackers.trim());
-                _util.setOpenTrackerString(openTrackers);
-                addMessage(_("Open Tracker list changed - torrent restart required to take effect."));
-                changed = true;
-            }
+        if (_util.shouldUseDHT() != useDHT) {
+            _config.setProperty(PROP_USE_DHT, Boolean.toString(useDHT));
+            if (useDHT)
+                addMessage(_("Enabled DHT."));
+            else
+                addMessage(_("Disabled DHT."));
+            if (_util.connected())
+                addMessage(_("DHT change requires tunnel shutdown and reopen"));
+            _util.setUseDHT(useDHT);
+            changed = true;
         }
         if (theme != null) {
             if(!theme.equals(_config.getProperty(PROP_THEME))) {
@@ -534,6 +641,84 @@ public class SnarkManager implements Snark.CompleteListener {
         }
     }
     
+    /**
+     *  Others should use the version in I2PSnarkUtil
+     *  @return non-null, empty if disabled
+     *  @since 0.9.1
+     */
+    private List<String> getOpenTrackers() {
+        if (!_util.shouldUseOpenTrackers())
+            return Collections.EMPTY_LIST;
+        return getListConfig(PROP_OPENTRACKERS, I2PSnarkUtil.DEFAULT_OPENTRACKERS);
+    }
+    
+    /**
+     *  @return non-null, fixed size, may be empty or unmodifiable
+     *  @since 0.9.1
+     */
+    public List<String> getPrivateTrackers() {
+        return getListConfig(PROP_PRIVATETRACKERS, null);
+    }
+
+    /**
+     *  @param ot null to restore default
+     *  @since 0.9.1
+     */
+    public void saveOpenTrackers(List<String> ot) {
+        String val = setListConfig(PROP_OPENTRACKERS, ot);
+        if (ot == null)
+            ot = Collections.singletonList(I2PSnarkUtil.DEFAULT_OPENTRACKERS);
+        _util.setOpenTrackers(ot);
+        addMessage(_("Open Tracker list changed - torrent restart required to take effect."));
+        saveConfig();
+    }
+
+    /**
+     *  @param pt null ok, default is none
+     *  @since 0.9.1
+     */
+    public void savePrivateTrackers(List<String> pt) {
+        setListConfig(PROP_PRIVATETRACKERS, pt);
+        addMessage(_("Private tracker list changed - affects newly created torrents only."));
+        saveConfig();
+    }
+
+    /**
+     *  @param dflt default or null
+     *  @return non-null, fixed size
+     *  @since 0.9.1
+     */
+    private List<String> getListConfig(String prop, String dflt) {
+        String val = _config.getProperty(prop);
+        if (val == null)
+            val = dflt;
+        if (val == null)
+            return Collections.EMPTY_LIST;
+        return Arrays.asList(val.split(","));
+    }
+
+    /**
+     *  Sets the config, does NOT save it
+     *  @param values may be null or empty
+     *  @return the comma-separated config string, non-null
+     *  @since 0.9.1
+     */
+    private String setListConfig(String prop, List<String> values) {
+        if (values == null || values.isEmpty()) {
+            _config.remove(prop);
+            return "";
+        }
+        StringBuilder buf = new StringBuilder(64);
+        for (String s : values) {
+             if (buf.length() > 0)
+                 buf.append(',');
+             buf.append(s);
+        }
+        String rv = buf.toString();
+        _config.setProperty(prop, rv);
+        return rv;
+    }
+
     public void saveConfig() {
         try {
             synchronized (_configFile) {
@@ -547,7 +732,7 @@ public class SnarkManager implements Snark.CompleteListener {
     public Properties getConfig() { return _config; }
     
     /** hardcoded for sanity.  perhaps this should be customizable, for people who increase their ulimit, etc. */
-    private static final int MAX_FILES_PER_TORRENT = 512;
+    public static final int MAX_FILES_PER_TORRENT = 512;
     
     /**
      *  Set of canonical .torrent filenames that we are dealing with.
@@ -563,6 +748,14 @@ public class SnarkManager implements Snark.CompleteListener {
      */
     public Snark getTorrent(String filename) { synchronized (_snarks) { return _snarks.get(filename); } }
 
+    /**
+     *  Unmodifiable
+     *  @since 0.9.4
+     */
+    public Collection<Snark> getTorrents() {
+        return Collections.unmodifiableCollection(_snarks.values());
+    }
+
     /**
      * Grab the torrent given the base name of the storage
      * @return Snark or null
@@ -623,7 +816,7 @@ public class SnarkManager implements Snark.CompleteListener {
         File dataDir = getDataDir();
         Snark torrent = null;
         synchronized (_snarks) {
-            torrent = (Snark)_snarks.get(filename);
+            torrent = _snarks.get(filename);
         }
         // don't hold the _snarks lock while verifying the torrent
         if (torrent == null) {
@@ -663,14 +856,16 @@ public class SnarkManager implements Snark.CompleteListener {
                     }
 
                     if (!TrackerClient.isValidAnnounce(info.getAnnounce())) {
-                        if (_util.shouldUseOpenTrackers() && _util.getOpenTrackers() != null) {
-                            //addMessage(_("Warning - No I2P trackers in \"{0}\", will announce to I2P open trackers and DHT only.", info.getName()));
-                            addMessage(_("Warning - No I2P trackers in \"{0}\", will announce to I2P open trackers only.", info.getName()));
-                        //} else if (_util.getDHT() != null) {
-                        //    addMessage(_("Warning - No I2P trackers in \"{0}\", and open trackers are disabled, will announce to DHT only.", info.getName()));
+                        if (info.isPrivate()) {
+                            addMessage(_("ERROR - No I2P trackers in private torrent \"{0}\"", info.getName()));
+                        } else if (!_util.getOpenTrackers().isEmpty()) {
+                            addMessage(_("Warning - No I2P trackers in \"{0}\", will announce to I2P open trackers and DHT only.", info.getName()));
+                            //addMessage(_("Warning - No I2P trackers in \"{0}\", will announce to I2P open trackers only.", info.getName()));
+                        } else if (_util.shouldUseDHT()) {
+                            addMessage(_("Warning - No I2P trackers in \"{0}\", and open trackers are disabled, will announce to DHT only.", info.getName()));
                         } else {
-                            //addMessage(_("Warning - No I2P trackers in \"{0}\", and DHT and open trackers are disabled, you should enable open trackers or DHT before starting the torrent.", info.getName()));
-                            addMessage(_("Warning - No I2P Trackers found in \"{0}\". Make sure Open Tracker is enabled before starting this torrent.", info.getName()));
+                            addMessage(_("Warning - No I2P trackers in \"{0}\", and DHT and open trackers are disabled, you should enable open trackers or DHT before starting the torrent.", info.getName()));
+                            //addMessage(_("Warning - No I2P Trackers found in \"{0}\". Make sure Open Tracker is enabled before starting this torrent.", info.getName()));
                             dontAutoStart = true;
                         }
                     }
@@ -691,7 +886,9 @@ public class SnarkManager implements Snark.CompleteListener {
                         }
                     }
                 } catch (IOException ioe) {
-                    addMessage(_("Torrent in \"{0}\" is invalid", sfile.getName()) + ": " + ioe.getMessage());
+                    String err = _("Torrent in \"{0}\" is invalid", sfile.getName()) + ": " + ioe.getMessage();
+                    addMessage(err);
+                    _log.error(err, ioe);
                     if (sfile.exists())
                         sfile.delete();
                     return;
@@ -727,7 +924,27 @@ public class SnarkManager implements Snark.CompleteListener {
      * @since 0.8.4
      */
     public void addMagnet(String name, byte[] ih, String trackerURL, boolean updateStatus) {
-        Snark torrent = new Snark(_util, name, ih, trackerURL, this,
+        addMagnet(name, ih, trackerURL, updateStatus, shouldAutoStart(), this);
+    }
+    
+    /**
+     * Add a torrent with the info hash alone (magnet / maggot)
+     * External use is for UpdateRunner.
+     *
+     * @param name hex or b32 name from the magnet link
+     * @param ih 20 byte info hash
+     * @param trackerURL may be null
+     * @param updateStatus should we add this magnet to the config file,
+     *                     to save it across restarts, in case we don't get
+     *                     the metadata before shutdown?
+     * @param listener to intercept callbacks, should pass through to this
+     * @return the new Snark or null on failure
+     * @throws RuntimeException via Snark.fatal()
+     * @since 0.9.4
+     */
+    public Snark addMagnet(String name, byte[] ih, String trackerURL, boolean updateStatus,
+                          boolean autoStart, CompleteListener listener) {
+        Snark torrent = new Snark(_util, name, ih, trackerURL, listener,
                                   _peerCoordinatorSet, _connectionAcceptor,
                                   false, getDataDir().getPath());
 
@@ -735,7 +952,7 @@ public class SnarkManager implements Snark.CompleteListener {
             Snark snark = getTorrentByInfoHash(ih);
             if (snark != null) {
                 addMessage(_("Torrent with this info hash is already running: {0}", snark.getBaseName()));
-                return;
+                return null;
             }
             // Tell the dir monitor not to delete us
             _magnets.add(name);
@@ -743,17 +960,21 @@ public class SnarkManager implements Snark.CompleteListener {
                 saveMagnetStatus(ih);
             _snarks.put(name, torrent);
         }
-        if (shouldAutoStart()) {
-            torrent.startTorrent();
+        if (autoStart) {
+            startTorrent(ih);
             addMessage(_("Fetching {0}", name));
-            boolean haveSavedPeers = false;
-            if ((!util().connected()) && !haveSavedPeers) {
-                addMessage(_("We have no saved peers and no other torrents are running. " +
-                             "Fetch of {0} will not succeed until you start another torrent.", name));
+            DHT dht = _util.getDHT();
+            boolean shouldWarn = _util.connected() &&
+                                 _util.getOpenTrackers().isEmpty() &&
+                                 ((!_util.shouldUseDHT()) || dht == null || dht.size() <= 0);
+            if (shouldWarn) {
+                addMessage(_("Open trackers are disabled and we have no DHT peers. " +
+                             "Fetch of {0} may not succeed until you start another torrent, enable open trackers, or enable DHT.", name));
             }
         } else {
             addMessage(_("Adding {0}", name));
-      }
+        }
+        return torrent;
     }
 
     /**
@@ -770,6 +991,29 @@ public class SnarkManager implements Snark.CompleteListener {
         _magnets.remove(snark.getName());
         removeMagnetStatus(snark.getInfoHash());
     }
+    
+    /**
+     * Add and start a FetchAndAdd task.
+     * Remove it with deleteMagnet().
+     *
+     * @param torrent must be instanceof FetchAndAdd
+     * @throws RuntimeException via Snark.fatal()?
+     * @since 0.9.1
+     */
+    public void addDownloader(Snark torrent) {
+        synchronized (_snarks) {
+            Snark snark = getTorrentByInfoHash(torrent.getInfoHash());
+            if (snark != null) {
+                addMessage(_("Download already running: {0}", snark.getBaseName()));
+                return;
+            }
+            String name = torrent.getName();
+            // Tell the dir monitor not to delete us
+            _magnets.add(name);
+            _snarks.put(name, torrent);
+        }
+        torrent.startTorrent();
+    }
 
     /**
      * Add a torrent from a MetaInfo. Save the MetaInfo data to filename.
@@ -1091,9 +1335,9 @@ public class SnarkManager implements Snark.CompleteListener {
         Snark torrent = null;
         synchronized (_snarks) {
             if (shouldRemove)
-                torrent = (Snark)_snarks.remove(filename);
+                torrent = _snarks.remove(filename);
             else
-                torrent = (Snark)_snarks.get(filename);
+                torrent = _snarks.get(filename);
             remaining = _snarks.size();
         }
         if (torrent != null) {
@@ -1152,13 +1396,11 @@ public class SnarkManager implements Snark.CompleteListener {
             // don't bother delaying if auto start is false
             long delay = 60 * 1000 * getStartupDelayMinutes();
             if (delay > 0 && shouldAutoStart()) {
-                _messages.add(_("Adding torrents in {0}", DataHelper.formatDuration2(delay)));
+                addMessage(_("Adding torrents in {0}", DataHelper.formatDuration2(delay)));
                 try { Thread.sleep(delay); } catch (InterruptedException ie) {}
-                // the first message was a "We are starting up in 1m" 
-                synchronized (_messages) { 
-                    if (_messages.size() == 1)
-                        _messages.remove(0);
-                }
+                // Remove that first message
+                if (_messages.size() == 1)
+                    _messages.poll();
             }
 
             // here because we need to delay until I2CP is up
@@ -1184,6 +1426,8 @@ public class SnarkManager implements Snark.CompleteListener {
                     } catch (Exception e) {
                         _log.error("Error in the DirectoryMonitor", e);
                     }
+                    if (!_snarks.isEmpty())
+                        addMessage(_("Up bandwidth limit is {0} KBps", _util.getMaxUpBW()));
                 }
                 try { Thread.sleep(60*1000); } catch (InterruptedException ie) {}
             }
@@ -1239,8 +1483,11 @@ public class SnarkManager implements Snark.CompleteListener {
                 return null;
             }
             saveTorrentStatus(meta, storage.getBitField(), null); // no file priorities
-            String name = (new File(getDataDir(), storage.getBaseName() + ".torrent")).getAbsolutePath();
+            // temp for addMessage() in case canonical throws
+            String name = storage.getBaseName();
             try {
+                // _snarks must use canonical
+                name = (new File(getDataDir(), storage.getBaseName() + ".torrent")).getCanonicalPath();
                 // put the announce URL in the file
                 String announce = snark.getTrackerURL();
                 if (announce != null)
@@ -1264,6 +1511,28 @@ public class SnarkManager implements Snark.CompleteListener {
         return null;
     }
 
+    /**
+     * A Snark.CompleteListener method.
+     * @since 0.9
+     */
+    public void fatal(Snark snark, String error) {
+        addMessage(_("Error on torrent {0}", snark.getName()) + ": " + error);
+    }
+    
+    /**
+     * A Snark.CompleteListener method.
+     * @since 0.9.2
+     */
+    public void addMessage(Snark snark, String message) {
+        addMessage(message);
+    }
+
+    /**
+     * A Snark.CompleteListener method.
+     * @since 0.9.4
+     */
+    public void gotPiece(Snark snark) {}
+
     // End Snark.CompleteListeners
 
     /**
@@ -1279,7 +1548,7 @@ public class SnarkManager implements Snark.CompleteListener {
                 byte[] ih = Base64.decode(b64);
                 // ignore value - TODO put tracker URL in value
                 if (ih != null && ih.length == 20)
-                    addMagnet("Magnet: " + I2PSnarkUtil.toHex(ih), ih, null, false);
+                    addMagnet(_("Magnet") + ' ' + I2PSnarkUtil.toHex(ih), ih, null, false);
                 // else remove from config?
             }
         }
@@ -1356,57 +1625,89 @@ public class SnarkManager implements Snark.CompleteListener {
     }
 
     /**
-     *  "name", "announceURL=websiteURL" pairs
+     *  Unsorted map of name to Tracker object
+     *  Modifiable, not a copy
+     *  @since 0.9.1
      */
-    private static final String DEFAULT_TRACKERS[] = { 
-//       "Postman", "http://YRgrgTLGnbTq2aZOZDJQ~o6Uk5k6TK-OZtx0St9pb0G-5EGYURZioxqYG8AQt~LgyyI~NCj6aYWpPO-150RcEvsfgXLR~CxkkZcVpgt6pns8SRc3Bi-QSAkXpJtloapRGcQfzTtwllokbdC-aMGpeDOjYLd8b5V9Im8wdCHYy7LRFxhEtGb~RL55DA8aYOgEXcTpr6RPPywbV~Qf3q5UK55el6Kex-6VCxreUnPEe4hmTAbqZNR7Fm0hpCiHKGoToRcygafpFqDw5frLXToYiqs9d4liyVB-BcOb0ihORbo0nS3CLmAwZGvdAP8BZ7cIYE3Z9IU9D1G8JCMxWarfKX1pix~6pIA-sp1gKlL1HhYhPMxwyxvuSqx34o3BqU7vdTYwWiLpGM~zU1~j9rHL7x60pVuYaXcFQDR4-QVy26b6Pt6BlAZoFmHhPcAuWfu-SFhjyZYsqzmEmHeYdAwa~HojSbofg0TMUgESRXMw6YThK1KXWeeJVeztGTz25sL8AAAA.i2p/announce.php=http://tracker.postman.i2p/"
-//       , "eBook", "http://E71FRom6PZNEqTN2Lr8P-sr23b7HJVC32KoGnVQjaX6zJiXwhJy2HsXob36Qmj81TYFZdewFZa9mSJ533UZgGyQkXo2ahctg82JKYZfDe5uDxAn1E9YPjxZCWJaFJh0S~UwSs~9AZ7UcauSJIoNtpxrtbmRNVFLqnkEDdLZi26TeucfOmiFmIWnVblLniWv3tG1boE9Abd-6j3FmYVrRucYuepAILYt6katmVNOk6sXmno1Eynrp~~MBuFq0Ko6~jsc2E2CRVYXDhGHEMdt-j6JUz5D7S2RIVzDRqQyAZLKJ7OdQDmI31przzmne1vOqqqLC~1xUumZVIvF~yOeJUGNjJ1Vx0J8i2BQIusn1pQJ6UCB~ZtZZLQtEb8EPVCfpeRi2ri1M5CyOuxN0V5ekmPHrYIBNevuTCRC26NP7ZS5VDgx1~NaC3A-CzJAE6f1QXi0wMI9aywNG5KGzOPifcsih8eyGyytvgLtrZtV7ykzYpPCS-rDfITncpn5hliPUAAAA.i2p/pub/bt/announce.php=http://de-ebook-archiv.i2p/pub/bt/"
-//       , "Gaytorrents", "http://uxPWHbK1OIj9HxquaXuhMiIvi21iK0~ZiG9d8G0840ZXIg0r6CbiV71xlsqmdnU6wm0T2LySriM0doW2gUigo-5BNkUquHwOjLROiETnB3ZR0Ml4IGa6QBPn1aAq2d9~g1r1nVjLE~pcFnXB~cNNS7kIhX1d6nLgYVZf0C2cZopEow2iWVUggGGnAA9mHjE86zLEnTvAyhbAMTqDQJhEuLa0ZYSORqzJDMkQt90MV4YMjX1ICY6RfUSFmxEqu0yWTrkHsTtRw48l~dz9wpIgc0a0T9C~eeWvmBFTqlJPtQZwntpNeH~jF7nlYzB58olgV2HHFYpVYD87DYNzTnmNWxCJ5AfDorm6AIUCV2qaE7tZtI1h6fbmGpGlPyW~Kw5GXrRfJwNvr6ajwAVi~bPVnrBwDZezHkfW4slOO8FACPR28EQvaTu9nwhAbqESxV2hCTq6vQSGjuxHeOuzBOEvRWkLKOHWTC09t2DbJ94FSqETmZopTB1ukEmaxRWbKSIaAAAA.i2p/announce.php=http://gaytorrents.i2p/"
-//       , "NickyB", "http://9On6d3cZ27JjwYCtyJJbowe054d5tFnfMjv4PHsYs-EQn4Y4mk2zRixatvuAyXz2MmRfXG-NAUfhKr0KCxRNZbvHmlckYfT-WBzwwpiMAl0wDFY~Pl8cqXuhfikSG5WrqdPfDNNIBuuznS0dqaczf~OyVaoEOpvuP3qV6wKqbSSLpjOwwAaQPHjlRtNIW8-EtUZp-I0LT45HSoowp~6b7zYmpIyoATvIP~sT0g0MTrczWhbVTUZnEkZeLhOR0Duw1-IRXI2KHPbA24wLO9LdpKKUXed05RTz0QklW5ROgR6TYv7aXFufX8kC0-DaKvQ5JKG~h8lcoHvm1RCzNqVE-2aiZnO2xH08H-iCWoLNJE-Td2kT-Tsc~3QdQcnEUcL5BF-VT~QYRld2--9r0gfGl-yDrJZrlrihHGr5J7ImahelNn9PpkVp6eIyABRmJHf2iicrk3CtjeG1j9OgTSwaNmEpUpn4aN7Kx0zNLdH7z6uTgCGD9Kmh1MFYrsoNlTp4AAAA.i2p/bittorrent/announce.php=http://nickyb.i2p/bittorrent/"
-//       , "Orion", "http://gKik1lMlRmuroXVGTZ~7v4Vez3L3ZSpddrGZBrxVriosCQf7iHu6CIk8t15BKsj~P0JJpxrofeuxtm7SCUAJEr0AIYSYw8XOmp35UfcRPQWyb1LsxUkMT4WqxAT3s1ClIICWlBu5An~q-Mm0VFlrYLIPBWlUFnfPR7jZ9uP5ZMSzTKSMYUWao3ejiykr~mtEmyls6g-ZbgKZawa9II4zjOy-hdxHgP-eXMDseFsrym4Gpxvy~3Fv9TuiSqhpgm~UeTo5YBfxn6~TahKtE~~sdCiSydqmKBhxAQ7uT9lda7xt96SS09OYMsIWxLeQUWhns-C~FjJPp1D~IuTrUpAFcVEGVL-BRMmdWbfOJEcWPZ~CBCQSO~VkuN1ebvIOr9JBerFMZSxZtFl8JwcrjCIBxeKPBmfh~xYh16BJm1BBBmN1fp2DKmZ2jBNkAmnUbjQOqWvUcehrykWk5lZbE7bjJMDFH48v3SXwRuDBiHZmSbsTY6zhGY~GkMQHNGxPMMSIAAAA.i2p/bt/announce.php=http://orion.i2p/bt/"
-//       , "anonymity", "http://8EoJZIKrWgGuDrxA3nRJs1jsPfiGwmFWL91hBrf0HA7oKhEvAna4Ocx47VLUR9retVEYBAyWFK-eZTPcvhnz9XffBEiJQQ~kFSCqb1fV6IfPiV3HySqi9U5Caf6~hC46fRd~vYnxmaBLICT3N160cxBETqH3v2rdxdJpvYt8q4nMk9LUeVXq7zqCTFLLG5ig1uKgNzBGe58iNcsvTEYlnbYcE930ABmrzj8G1qQSgSwJ6wx3tUQNl1z~4wSOUMan~raZQD60lRK70GISjoX0-D0Po9WmPveN3ES3g72TIET3zc3WPdK2~lgmKGIs8GgNLES1cXTolvbPhdZK1gxddRMbJl6Y6IPFyQ9o4-6Rt3Lp-RMRWZ2TG7j2OMcNSiOmATUhKEFBDfv-~SODDyopGBmfeLw16F4NnYednvn4qP10dyMHcUASU6Zag4mfc2-WivrOqeWhD16fVAh8MoDpIIT~0r9XmwdaVFyLcjbXObabJczxCAW3fodQUnvuSkwzAAAA.i2p/anonymityTracker/announce.php=http://anonymityweb.i2p/anonymityTracker/"
-//       , "The freak's tracker", "http://mHKva9x24E5Ygfey2llR1KyQHv5f8hhMpDMwJDg1U-hABpJ2NrQJd6azirdfaR0OKt4jDlmP2o4Qx0H598~AteyD~RJU~xcWYdcOE0dmJ2e9Y8-HY51ie0B1yD9FtIV72ZI-V3TzFDcs6nkdX9b81DwrAwwFzx0EfNvK1GLVWl59Ow85muoRTBA1q8SsZImxdyZ-TApTVlMYIQbdI4iQRwU9OmmtefrCe~ZOf4UBS9-KvNIqUL0XeBSqm0OU1jq-D10Ykg6KfqvuPnBYT1BYHFDQJXW5DdPKwcaQE4MtAdSGmj1epDoaEBUa9btQlFsM2l9Cyn1hzxqNWXELmx8dRlomQLlV4b586dRzW~fLlOPIGC13ntPXogvYvHVyEyptXkv890jC7DZNHyxZd5cyrKC36r9huKvhQAmNABT2Y~pOGwVrb~RpPwT0tBuPZ3lHYhBFYmD8y~AOhhNHKMLzea1rfwTvovBMByDdFps54gMN1mX4MbCGT4w70vIopS9yAAAA.i2p/bytemonsoon/announce.php"
-//       , "mastertracker", "http://VzXD~stRKbL3MOmeTn1iaCQ0CFyTmuFHiKYyo0Rd~dFPZFCYH-22rT8JD7i-C2xzYFa4jT5U2aqHzHI-Jre4HL3Ri5hFtZrLk2ax3ji7Qfb6qPnuYkuiF2E2UDmKUOppI8d9Ye7tjdhQVCy0izn55tBaB-U7UWdcvSK2i85sauyw3G0Gfads1Rvy5-CAe2paqyYATcDmGjpUNLoxbfv9KH1KmwRTNH6k1v4PyWYYnhbT39WfKMbBjSxVQRdi19cyJrULSWhjxaQfJHeWx5Z8Ev4bSPByBeQBFl2~4vqy0S5RypINsRSa3MZdbiAAyn5tr5slWR6QdoqY3qBQgBJFZppy-3iWkFqqKgSxCPundF8gdDLC5ddizl~KYcYKl42y9SGFHIukH-TZs8~em0~iahzsqWVRks3zRG~tlBcX2U3M2~OJs~C33-NKhyfZT7-XFBREvb8Szmd~p66jDxrwOnKaku-G6DyoQipJqIz4VHmY9-y5T8RrUcJcM-5lVoMpAAAA.i2p/announce.php=http://tracker.mastertracker.i2p/"
-//       , "Galen", "http://5jpwQMI5FT303YwKa5Rd38PYSX04pbIKgTaKQsWbqoWjIfoancFdWCShXHLI5G5ofOb0Xu11vl2VEMyPsg1jUFYSVnu4-VfMe3y4TKTR6DTpetWrnmEK6m2UXh91J5DZJAKlgmO7UdsFlBkQfR2rY853-DfbJtQIFl91tbsmjcA5CGQi4VxMFyIkBzv-pCsuLQiZqOwWasTlnzey8GcDAPG1LDcvfflGV~6F5no9mnuisZPteZKlrv~~TDoXTj74QjByWc4EOYlwqK8sbU9aOvz~s31XzErbPTfwiawiaZ0RUI-IDrKgyvmj0neuFTWgjRGVTH8bz7cBZIc3viy6ioD-eMQOrXaQL0TCWZUelRwHRvgdPiQrxdYQs7ixkajeHzxi-Pq0EMm5Vbh3j3Q9kfUFW3JjFDA-MLB4g6XnjCbM5J1rC0oOBDCIEfhQkszru5cyLjHiZ5yeA0VThgu~c7xKHybv~OMXION7V8pBKOgET7ZgAkw1xgYe3Kkyq5syAAAA.i2p/tr/announce.php=http://galen.i2p/tr/"
-       "Postman", "http://tracker2.postman.i2p/announce.php=http://tracker2.postman.i2p/"
-       ,"Welterde", "http://tracker.welterde.i2p/a=http://tracker.welterde.i2p/stats?mode=top5"
-       ,"Diftracker", "http://n--XWjHjUPjnMNrSwXA2OYXpMIUL~u4FNXnrt2HtjK3y6j~4SOClyyeKzd0zRPlixxkCe2wfBIYye3bZsaqAD8bd0QMmowxbq91WpjsPfKMiphJbePKXtYAVARiy0cqyvh1d2LyDE-6wkvgaw45hknmS0U-Dg3YTJZbAQRU2SKXgIlAbWCv4R0kDFBLEVpReDiJef3rzAWHiW8yjmJuJilkYjMwlfRjw8xx1nl2s~yhlljk1pl13jGYb0nfawQnuOWeP-ASQWvAAyVgKvZRJE2O43S7iveu9piuv7plXWbt36ef7ndu2GNoNyPOBdpo9KUZ-NOXm4Kgh659YtEibL15dEPAOdxprY0sYUurVw8OIWqrpX7yn08nbi6qHVGqQwTpxH35vkL8qrCbm-ym7oQJQnNmSDrNTyWYRFSq5s5~7DAdFDzqRPW-pX~g0zEivWj5tzkhvG9rVFgFo0bpQX3X0PUAV9Xbyf8u~v8Zbr9K1pCPqBq9XEr4TqaLHw~bfAAAA.i2p/announce.php=http://diftracker.i2p/"
-//       , "CRSTRACK", "http://b4G9sCdtfvccMAXh~SaZrPqVQNyGQbhbYMbw6supq2XGzbjU4NcOmjFI0vxQ8w1L05twmkOvg5QERcX6Mi8NQrWnR0stLExu2LucUXg1aYjnggxIR8TIOGygZVIMV3STKH4UQXD--wz0BUrqaLxPhrm2Eh9Hwc8TdB6Na4ShQUq5Xm8D4elzNUVdpM~RtChEyJWuQvoGAHY3ppX-EJJLkiSr1t77neS4Lc-KofMVmgI9a2tSSpNAagBiNI6Ak9L1T0F9uxeDfEG9bBSQPNMOSUbAoEcNxtt7xOW~cNOAyMyGydwPMnrQ5kIYPY8Pd3XudEko970vE0D6gO19yoBMJpKx6Dh50DGgybLQ9CpRaynh2zPULTHxm8rneOGRcQo8D3mE7FQ92m54~SvfjXjD2TwAVGI~ae~n9HDxt8uxOecAAvjjJ3TD4XM63Q9TmB38RmGNzNLDBQMEmJFpqQU8YeuhnS54IVdUoVQFqui5SfDeLXlSkh4vYoMU66pvBfWbAAAA.i2p/tracker/announce.php=http://crstrack.i2p/tracker/"
-//       ,"Exotrack", "http://blbgywsjubw3d2zih2giokakhe3o2cko7jtte4risb3hohbcoyva.b32.i2p/announce.php=http://exotrack.i2p/"
-    };
-    
-    /** comma delimited list of name=announceURL=baseURL for the trackers to be displayed */
-    public static final String PROP_TRACKERS = "i2psnark.trackers";
-    private static Map<String, String> trackerMap = null;
-    /** sorted map of name to announceURL=baseURL */
-    public Map<String, String> getTrackers() { 
-        if (trackerMap != null) // only do this once, can't be updated while running
-            return trackerMap;
-        Map<String, String> rv = new TreeMap();
+    public Map<String, Tracker> getTrackerMap() { 
+        return _trackerMap;
+    }
+
+    /**
+     *  Unsorted, do not modify
+     */
+    public Collection<Tracker> getTrackers() { 
+        return _trackerMap.values();
+    }
+
+    /**
+     *  Sorted copy
+     *  @since 0.9.1
+     */
+    public List<Tracker> getSortedTrackers() { 
+        List<Tracker> rv = new ArrayList(_trackerMap.values());
+        Collections.sort(rv, new IgnoreCaseComparator());
+        return rv;
+    }
+
+    /** @since 0.9 */
+    private void initTrackerMap() {
         String trackers = _config.getProperty(PROP_TRACKERS);
         if ( (trackers == null) || (trackers.trim().length() <= 0) )
             trackers = _context.getProperty(PROP_TRACKERS);
         if ( (trackers == null) || (trackers.trim().length() <= 0) ) {
-            for (int i = 0; i < DEFAULT_TRACKERS.length; i += 2)
-                rv.put(DEFAULT_TRACKERS[i], DEFAULT_TRACKERS[i+1]);
+            setDefaultTrackerMap(true);
         } else {
-            StringTokenizer tok = new StringTokenizer(trackers, ",");
-            while (tok.hasMoreTokens()) {
-                String pair = tok.nextToken();
-                int split = pair.indexOf('=');
-                if (split <= 0)
-                    continue;
-                String name = pair.substring(0, split).trim();
-                String url = pair.substring(split+1).trim();
-                if ( (name.length() > 0) && (url.length() > 0) )
-                    rv.put(name, url);
+            String[] toks = trackers.split(",");
+            for (int i = 0; i < toks.length; i += 2) {
+                String name = toks[i].trim().replace("&#44;", ",");
+                String url = toks[i+1].trim().replace("&#44;", ",");
+                if ( (name.length() > 0) && (url.length() > 0) ) {
+                    String urls[] = url.split("=", 2);
+                    String url2 = urls.length > 1 ? urls[1] : "";
+                    _trackerMap.put(name, new Tracker(name, urls[0], url2));
+                }
             }
         }
-        
-        trackerMap = rv;
-        return trackerMap;
     }
-    
+
+    /** @since 0.9 */
+    public void setDefaultTrackerMap() {
+        setDefaultTrackerMap(true);
+    }
+
+    /** @since 0.9.1 */
+    private void setDefaultTrackerMap(boolean save) {
+        _trackerMap.clear();
+        for (int i = 0; i < DEFAULT_TRACKERS.length; i += 2) {
+            String name = DEFAULT_TRACKERS[i];
+            String urls[] = DEFAULT_TRACKERS[i+1].split("=", 2);
+            String url2 = urls.length > 1 ? urls[1] : null;
+            _trackerMap.put(name, new Tracker(name, urls[0], url2));
+        }
+        if (save && _config.remove(PROP_TRACKERS) != null) {
+            saveConfig();
+        }
+    }
+
+    /** @since 0.9 */
+    public void saveTrackerMap() {
+        StringBuilder buf = new StringBuilder(2048);
+        boolean comma = false;
+        for (Map.Entry<String, Tracker> e : _trackerMap.entrySet()) {
+            if (comma)
+                buf.append(',');
+            else
+                comma = true;
+            Tracker t = e.getValue();
+            buf.append(e.getKey().replace(",", "&#44;")).append(',').append(t.announceURL.replace(",", "&#44;"));
+            if (t.baseURL != null)
+                buf.append('=').append(t.baseURL);
+        }
+        _config.setProperty(PROP_TRACKERS, buf.toString());
+        saveConfig();
+    }
+
     private static class TorrentFilenameFilter implements FilenameFilter {
         private static final TorrentFilenameFilter _filter = new TorrentFilenameFilter();
         public static TorrentFilenameFilter instance() { return _filter; }
@@ -1415,15 +1716,147 @@ public class SnarkManager implements Snark.CompleteListener {
         }
     }
 
-    public class SnarkManagerShutdown extends I2PAppThread {
-        @Override
+    /**
+     *  If not connected, thread it, otherwise inline
+     *  @since 0.9.1
+     */
+    public void startTorrent(byte[] infoHash) {
+        for (Snark snark : _snarks.values()) {
+            if (DataHelper.eq(infoHash, snark.getInfoHash())) {
+                if (snark.isStarting() || !snark.isStopped()) {
+                    addMessage("Torrent already started");
+                    return;
+                }
+                boolean connected = _util.connected();
+                if ((!connected) && !_util.isConnecting())
+                    addMessage(_("Opening the I2P tunnel"));
+                addMessage(_("Starting up torrent {0}", snark.getBaseName()));
+                if (connected) {
+                    snark.startTorrent();
+                } else {
+                    // mark it for the UI
+                    snark.setStarting();
+                    (new I2PAppThread(new ThreadedStarter(snark), "TorrentStarter", true)).start();
+                    try { Thread.sleep(200); } catch (InterruptedException ie) {}
+                }
+                return;
+            }
+        }
+        addMessage("Torrent not found?");
+    }
+
+    /**
+     *  If not connected, thread it, otherwise inline
+     *  @since 0.9.1
+     */
+    public void startAllTorrents() {
+        if (_util.connected()) {
+            startAll();
+        } else {
+            addMessage(_("Opening the I2P tunnel and starting all torrents."));
+            for (Snark snark : _snarks.values()) {
+                // mark it for the UI
+                snark.setStarting();
+            }
+            (new I2PAppThread(new ThreadedStarter(null), "TorrentStarterAll", true)).start();
+            try { Thread.sleep(200); } catch (InterruptedException ie) {}
+        }
+    }
+
+    /**
+     *  Use null constructor param for all
+     *  @since 0.9.1
+     */
+    private class ThreadedStarter implements Runnable {
+        private final Snark snark;
+        public ThreadedStarter(Snark s) { snark = s; }
         public void run() {
-            Set names = listTorrentFiles();
-            for (Iterator iter = names.iterator(); iter.hasNext(); ) {
-                Snark snark = getTorrent((String)iter.next());
-                if ( (snark != null) && (!snark.isStopped()) )
-                    snark.stopTorrent();
+            if (snark != null) {
+                if (snark.isStopped())
+                    snark.startTorrent();
+            } else {
+                startAll();
             }
         }
     }
+
+    /**
+     *  Inline
+     *  @since 0.9.1
+     */
+    private void startAll() {
+        for (Snark snark : _snarks.values()) {
+            if (snark.isStopped())
+                snark.startTorrent();
+        }
+    }
+
+    /**
+     * Stop all running torrents, and close the tunnel after a delay
+     * to allow for announces.
+     * If called at router shutdown via Jetty shutdown hook -> webapp destroy() -> stop(),
+     * the tunnel won't actually be closed as the SimpleScheduler is already shutdown
+     * or will be soon, so we delay a few seconds inline.
+     * @param finalShutdown if true, sleep at the end if any torrents were running
+     * @since 0.9.1
+     */
+    public void stopAllTorrents(boolean finalShutdown) {
+        _stopping = true;
+        if (finalShutdown && _log.shouldLog(Log.WARN))
+            _log.warn("SnarkManager final shutdown");
+        int count = 0;
+        for (Snark snark : _snarks.values()) {
+            if (!snark.isStopped()) {
+                if (count == 0)
+                    addMessage(_("Stopping all torrents and closing the I2P tunnel."));
+                count++;
+                if (finalShutdown)
+                    snark.stopTorrent(true);
+                else
+                    stopTorrent(snark, false);
+                // Throttle since every unannounce is now threaded.
+                // How to do this without creating a ton of threads?
+                try { Thread.sleep(20); } catch (InterruptedException ie) {}
+            }
+        }
+        if (_util.connected()) {
+            if (count > 0) {
+                DHT dht = _util.getDHT();
+                if (dht != null)
+                    dht.stop();
+                // Schedule this even for final shutdown, as there's a chance
+                // that it's just this webapp that is stopping.
+                _context.simpleScheduler().addEvent(new Disconnector(), 60*1000);
+                addMessage(_("Closing I2P tunnel after notifying trackers."));
+                if (finalShutdown) {
+                    try { Thread.sleep(5*1000); } catch (InterruptedException ie) {}
+                }
+            } else {
+                _util.disconnect();
+                _stopping = false;
+                addMessage(_("I2P tunnel closed."));
+            }
+        }
+    }
+
+    /** @since 0.9.1 */
+    private class Disconnector implements SimpleTimer.TimedEvent {
+        public void timeReached() {
+            if (_util.connected()) {
+                _util.disconnect();
+                _stopping = false;
+                addMessage(_("I2P tunnel closed."));
+            }
+        }
+    }
+    
+    /**
+     *  ignore case, current locale
+     *  @since 0.9
+     */
+    private static class IgnoreCaseComparator implements Comparator<Tracker> {
+        public int compare(Tracker l, Tracker r) {
+            return l.name.toLowerCase().compareTo(r.name.toLowerCase());
+        }
+    }
 }

apps/i2psnark/java/src/org/klomp/snark/SnarkShutdown.java

@@ -26,6 +26,7 @@ import net.i2p.util.I2PAppThread;
 
 /**
  * Makes sure everything ends correctly when shutting down.
+ * @deprecated unused
  */
 public class SnarkShutdown extends I2PAppThread
 {
@@ -61,7 +62,7 @@ public class SnarkShutdown extends I2PAppThread
 
     //Snark.debug("Halting TrackerClient...", Snark.INFO);
     if (trackerclient != null)
-      trackerclient.halt();
+      trackerclient.halt(true);
 
     //Snark.debug("Halting PeerCoordinator...", Snark.INFO);
     if (coordinator != null)

apps/i2psnark/java/src/org/klomp/snark/Storage.java

@@ -21,6 +21,7 @@
 package org.klomp.snark;
 
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.RandomAccessFile;
 import java.nio.charset.Charset;
@@ -32,16 +33,23 @@ import java.util.List;
 import java.util.Map;
 import java.util.StringTokenizer;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
 
+import net.i2p.I2PAppContext;
 import net.i2p.crypto.SHA1;
+import net.i2p.data.ByteArray;
+import net.i2p.data.DataHelper;
+import net.i2p.util.ByteCache;
+import net.i2p.util.Log;
 import net.i2p.util.SecureFile;
+import net.i2p.util.SystemVersion;
 
 /**
  * Maintains pieces on disk. Can be used to store and retrieve pieces.
  */
 public class Storage
 {
-  private MetaInfo metainfo;
+  private final MetaInfo metainfo;
   private long[] lengths;
   private RandomAccessFile[] rafs;
   private String[] names;
@@ -50,9 +58,12 @@ public class Storage
   private File[] RAFfile;    // File to make it easier to reopen
   /** priorities by file; default 0; may be null. @since 0.8.1 */
   private int[] priorities;
+  /** is the file empty and sparse? */
+  private boolean[] isSparse;
 
   private final StorageListener listener;
   private final I2PSnarkUtil _util;
+  private final Log _log;
 
   private /* FIXME final FIXME */ BitField bitfield; // BitField to represent the pieces
   private int needed; // Number of pieces needed
@@ -62,6 +73,8 @@ public class Storage
   private final int pieces;
   private final long total_length;
   private boolean changed;
+  private volatile boolean _isChecking;
+  private final AtomicInteger _allocateCount = new AtomicInteger();
 
   /** The default piece size. */
   private static final int MIN_PIECE_SIZE = 256*1024;
@@ -73,20 +86,24 @@ public class Storage
 
   private static final Map<String, String> _filterNameCache = new ConcurrentHashMap();
 
+  private static final boolean _isWindows = SystemVersion.isWindows();
+
+  private static final int BUFSIZE = PeerState.PARTSIZE;
+  private static final ByteCache _cache = ByteCache.getInstance(16, BUFSIZE);
+
   /**
    * Creates a new storage based on the supplied MetaInfo.  This will
    * try to create and/or check all needed files in the MetaInfo.
    *
-   * @exception IOException when creating and/or checking files fails.
+   * Does not check storage. Caller MUST call check()
    */
   public Storage(I2PSnarkUtil util, MetaInfo metainfo, StorageListener listener)
-    throws IOException
   {
     _util = util;
+    _log = util.getContext().logManager().getLog(Storage.class);
     this.metainfo = metainfo;
     this.listener = listener;
     needed = metainfo.getPieces();
-    _probablyComplete = false;
     bitfield = new BitField(needed);
     piece_size = metainfo.getPieceLength(0);
     pieces = needed;
@@ -94,17 +111,23 @@ public class Storage
   }
 
   /**
-   * Creates a storage from the existing file or directory together
-   * with an appropriate MetaInfo file as can be announced on the
-   * given announce String location.
+   * Creates a storage from the existing file or directory.
+   * Creates an in-memory metainfo but does not save it to
+   * a file, caller must do that.
+   *
+   * Creates the metainfo, this may take a LONG time. BLOCKING.
    *
    * @param announce may be null
    * @param listener may be null
+   * @throws IOException when creating and/or checking files fails.
    */
-  public Storage(I2PSnarkUtil util, File baseFile, String announce, StorageListener listener)
+  public Storage(I2PSnarkUtil util, File baseFile, String announce,
+                 List<List<String>> announce_list,
+                 boolean privateTorrent, StorageListener listener)
     throws IOException
   {
     _util = util;
+    _log = util.getContext().logManager().getLog(Storage.class);
     this.listener = listener;
     // Create names, rafs and lengths arrays.
     getFiles(baseFile);
@@ -120,6 +143,8 @@ public class Storage
 
     if (total <= 0)
         throw new IOException("Torrent contains no data");
+    if (total > MAX_TOTAL_SIZE)
+        throw new IOException("Torrent too big (" + total + " bytes), max is " + MAX_TOTAL_SIZE);
 
     int pc_size = MIN_PIECE_SIZE;
     int pcs = (int) ((total - 1)/pc_size) + 1;
@@ -155,9 +180,11 @@ public class Storage
         lengthsList = null;
       }
 
+    // TODO thread this so we can return and show something on the UI
     byte[] piece_hashes = fast_digestCreate();
     metainfo = new MetaInfo(announce, baseFile.getName(), null, files,
-                            lengthsList, piece_size, piece_hashes, total);
+                            lengthsList, piece_size, piece_hashes, total, privateTorrent,
+                            announce_list);
 
   }
 
@@ -190,6 +217,8 @@ public class Storage
 
   private void getFiles(File base) throws IOException
   {
+    if (base.getAbsolutePath().equals("/"))
+        throw new IOException("Don't seed root");
     ArrayList files = new ArrayList();
     addFiles(files, base);
 
@@ -201,7 +230,7 @@ public class Storage
     RAFtime = new long[size];
     RAFfile = new File[size];
     priorities = new int[size];
-
+    isSparse = new boolean[size];
 
     int i = 0;
     Iterator it = files.iterator();
@@ -218,17 +247,21 @@ public class Storage
       }
   }
 
-  private void addFiles(List l, File f)
-  {
-    if (!f.isDirectory())
-      l.add(f);
-    else
-      {
+  /**
+   *  @throws IOException if too many total files
+   */
+  private void addFiles(List l, File f) throws IOException {
+    if (!f.isDirectory()) {
+        if (l.size() >= SnarkManager.MAX_FILES_PER_TORRENT)
+            throw new IOException("Too many files, limit is " + SnarkManager.MAX_FILES_PER_TORRENT + ", zip them?");
+        l.add(f);
+    } else {
         File[] files = f.listFiles();
         if (files == null)
           {
-            _util.debug("WARNING: Skipping '" + f 
-                        + "' not a normal file.", Snark.WARNING);
+            if (_log.shouldLog(Log.WARN))
+                _log.warn("WARNING: Skipping '" + f 
+                        + "' not a normal file.");
             return;
           }
         for (int i = 0; i < files.length; i++)
@@ -268,6 +301,23 @@ public class Storage
       return changed;
   }
 
+  /**
+   *  File checking in progress.
+   *  @since 0.9.3
+   */
+  public boolean isChecking() {
+      return _isChecking;
+  }
+
+  /**
+   *  Disk allocation (ballooning) in progress.
+   *  Always false on Windows.
+   *  @since 0.9.3
+   */
+  public boolean isAllocating() {
+      return _allocateCount.get() > 0;
+  }
+
   /**
    *  @param file canonical path (non-directory)
    *  @return number of bytes remaining; -1 if unknown file
@@ -333,7 +383,8 @@ public class Storage
   }
 
   /**
-   *  Must call setPiecePriorities() after calling this
+   *  Must call Snark.updatePiecePriorities()
+   *  (which calls getPiecePriorities()) after calling this.
    *  @param file canonical path (non-directory)
    *  @param pri default 0; <0 to disable
    *  @since 0.8.1
@@ -451,7 +502,8 @@ public class Storage
     if (files == null)
       {
         // Create base as file.
-        _util.debug("Creating/Checking file: " + base, Snark.NOTICE);
+        if (_log.shouldLog(Log.INFO))
+            _log.info("Creating/Checking file: " + base);
         if (!base.createNewFile() && !base.exists())
           throw new IOException("Could not create file " + base);
 
@@ -461,6 +513,7 @@ public class Storage
         RAFlock = new Object[1];
         RAFtime = new long[1];
         RAFfile = new File[1];
+        isSparse = new boolean[1];
         lengths[0] = metainfo.getTotalLength();
         RAFlock[0] = new Object();
         RAFfile[0] = base;
@@ -474,7 +527,8 @@ public class Storage
     else
       {
         // Create base as dir.
-        _util.debug("Creating/Checking directory: " + base, Snark.NOTICE);
+        if (_log.shouldLog(Log.INFO))
+            _log.info("Creating/Checking directory: " + base);
         if (!base.mkdir() && !base.isDirectory())
           throw new IOException("Could not create directory " + base);
 
@@ -487,6 +541,7 @@ public class Storage
         RAFlock = new Object[size];
         RAFtime = new long[size];
         RAFfile = new File[size];
+        isSparse = new boolean[size];
         for (int i = 0; i < size; i++)
           {
             List<String> path = files.get(i);
@@ -532,19 +587,22 @@ public class Storage
       bitfield = savedBitField;
       needed = metainfo.getPieces() - bitfield.count();
       _probablyComplete = complete();
-      _util.debug("Found saved state and files unchanged, skipping check", Snark.NOTICE);
+      if (_log.shouldLog(Log.INFO))
+          _log.info("Found saved state and files unchanged, skipping check");
     } else {
       // the following sets the needed variable
       changed = true;
       checkCreateFiles(false);
     }
     if (complete()) {
-        _util.debug("Torrent is complete", Snark.NOTICE);
+        if (_log.shouldLog(Log.INFO))
+            _log.info("Torrent is complete");
     } else {
         // fixme saved priorities
         if (files != null)
             priorities = new int[files.size()];
-        _util.debug("Still need " + needed + " out of " + metainfo.getPieces() + " pieces", Snark.NOTICE);
+        if (_log.shouldLog(Log.INFO))
+            _log.info("Still need " + needed + " out of " + metainfo.getPieces() + " pieces");
     }
   }
 
@@ -679,11 +737,25 @@ public class Storage
    * This is called at the beginning, and at presumed completion,
    * so we have to be careful about locking.
    *
+   * TODO thread the checking so we can return and display
+   * something on the UI
+   *
    * @param recheck if true, this is a check after we downloaded the
    *        last piece, and we don't modify the global bitfield unless
    *        the check fails.
    */
-  private void checkCreateFiles(boolean recheck) throws IOException
+  private void checkCreateFiles(boolean recheck) throws IOException {
+      synchronized(this) {
+          _isChecking = true;
+          try {
+              locked_checkCreateFiles(recheck);
+          } finally {
+              _isChecking = false;
+          }
+      }
+  }
+
+  private void locked_checkCreateFiles(boolean recheck) throws IOException
   {
     // Whether we are resuming or not,
     // if any of the files already exists we assume we are resuming.
@@ -700,6 +772,7 @@ public class Storage
     }
 
     // Make sure all files are available and of correct length
+    // The files should all exist as they have been created with zero length by createFilesFromNames()
     for (int i = 0; i < rafs.length; i++)
       {
         long length = RAFfile[i].length();
@@ -721,9 +794,11 @@ public class Storage
         } else {
           String msg = "File '" + names[i] + "' exists, but has wrong length (expected " +
                        lengths[i] + " but found " + length + ") - repairing corruption";
-          SnarkManager.instance().addMessage(msg);
-          _util.debug(msg, Snark.ERROR);
+          if (listener != null)
+              listener.addMessage(msg);
+          _log.error(msg);
           changed = true;
+          resume = true;
           _probablyComplete = false; // to force RW
           synchronized(RAFlock[i]) {
               checkRAF(i);
@@ -797,28 +872,66 @@ public class Storage
     }
   }
 
-  /** this calls openRAF(); caller must synnchronize and call closeRAF() */
+  /**
+   *  This creates a (presumably) sparse file so that reads won't fail with IOE.
+   *  Sets isSparse[nr] = true. balloonFile(nr) should be called later to
+   *  defrag the file.
+   *
+   *  This calls openRAF(); caller must synchronize and call closeRAF().
+   */
   private void allocateFile(int nr) throws IOException
   {
     // caller synchronized
     openRAF(nr, false);  // RW
-    // XXX - Is this the best way to make sure we have enough space for
-    // the whole file?
     long remaining = lengths[nr];
     if (listener != null)
         listener.storageCreateFile(this, names[nr], remaining);
-    final int ZEROBLOCKSIZE = (int) Math.min(remaining, 32*1024);
-    byte[] zeros = new byte[ZEROBLOCKSIZE];
-    while (remaining > 0) {
-        int size = (int) Math.min(remaining, ZEROBLOCKSIZE);
-        rafs[nr].write(zeros, 0, size);
-        remaining -= size;
-    }
+    rafs[nr].setLength(remaining);
+    // don't bother ballooning later on Windows since there is no sparse file support
+    // until JDK7 using the JSR-203 interface.
+    // RAF seeks/writes do not create sparse files.
+    // Windows will zero-fill up to the point of the write, which
+    // will make the file fairly unfragmented, on average, at least until
+    // near the end where it will get exponentially more fragmented.
+    if (!_isWindows)
+        isSparse[nr] = true;
     // caller will close rafs[nr]
     if (listener != null)
       listener.storageAllocated(this, lengths[nr]);
   }
 
+  /**
+   *  This "balloons" the file with zeros to eliminate disk fragmentation.,
+   *  Overwrites the entire file with zeros. Sets isSparse[nr] = false.
+   *
+   *  Caller must synchronize and call checkRAF() or openRAF().
+   *  @since 0.9.1
+   */
+  private void balloonFile(int nr) throws IOException
+  {
+    if (_log.shouldLog(Log.INFO))
+        _log.info("Ballooning " + nr + ": " + RAFfile[nr]);
+    long remaining = lengths[nr];
+    final int ZEROBLOCKSIZE = (int) Math.min(remaining, 32*1024);
+    byte[] zeros = new byte[ZEROBLOCKSIZE];
+    rafs[nr].seek(0);
+    // don't bother setting flag for small files
+    if (remaining > 20*1024*1024)
+        _allocateCount.incrementAndGet();
+    try {
+        while (remaining > 0) {
+            int size = (int) Math.min(remaining, ZEROBLOCKSIZE);
+            rafs[nr].write(zeros, 0, size);
+            remaining -= size;
+        }
+    } finally {
+        remaining = lengths[nr];
+        if (remaining > 20*1024*1024)
+            _allocateCount.decrementAndGet();
+    }
+    isSparse[nr] = false;
+  }
+
 
   /**
    * Closes the Storage and makes sure that all RandomAccessFiles are
@@ -837,7 +950,7 @@ public class Storage
             closeRAF(i);
           }
         } catch (IOException ioe) {
-            _util.debug("Error closing " + RAFfile[i], Snark.ERROR, ioe);
+            _log.error("Error closing " + RAFfile[i], ioe);
             // gobble gobble
         }
       }
@@ -848,78 +961,99 @@ public class Storage
    * Returns a byte array containing a portion of the requested piece or null if
    * the storage doesn't contain the piece yet.
    */
-  public byte[] getPiece(int piece, int off, int len) throws IOException
+  public ByteArray getPiece(int piece, int off, int len) throws IOException
   {
     if (!bitfield.get(piece))
       return null;
 
     //Catch a common place for OOMs esp. on 1MB pieces
+    ByteArray rv;
     byte[] bs;
     try {
-      bs = new byte[len];
+        // Will be restored to cache in Message.sendMessage()
+        if (len == BUFSIZE)
+            rv = _cache.acquire();
+        else
+            rv = new ByteArray(new byte[len]);
     } catch (OutOfMemoryError oom) {
-      _util.debug("Out of memory, can't honor request for piece " + piece, Snark.WARNING, oom);
+      if (_log.shouldLog(Log.WARN))
+          _log.warn("Out of memory, can't honor request for piece " + piece, oom);
       return null;
     }
+    bs = rv.getData();
     getUncheckedPiece(piece, bs, off, len);
-    return bs;
+    return rv;
   }
 
   /**
    * Put the piece in the Storage if it is correct.
+   * Warning - takes a LONG time if complete as it does the recheck here.
+   * TODO thread the recheck?
    *
    * @return true if the piece was correct (sha metainfo hash
    * matches), otherwise false.
    * @exception IOException when some storage related error occurs.
    */
-  public boolean putPiece(int piece, byte[] ba) throws IOException
+  public boolean putPiece(PartialPiece pp) throws IOException
   {
-    // First check if the piece is correct.
-    // Copy the array first to be paranoid.
-    byte[] bs = (byte[]) ba.clone();
-    int length = bs.length;
-    boolean correctHash = metainfo.checkPiece(piece, bs, 0, length);
-    if (listener != null)
-      listener.storageChecked(this, piece, correctHash);
-    if (!correctHash)
-      return false;
+      int piece = pp.getPiece();
+      try {
+          synchronized(bitfield) {
+              if (bitfield.get(piece))
+                  return true; // No need to store twice.
+          }
 
-    synchronized(bitfield)
-      {
-        if (bitfield.get(piece))
-          return true; // No need to store twice.
-      }
+          // TODO alternative - check hash on the fly as we write to the file,
+          // to save another I/O pass
+          boolean correctHash = metainfo.checkPiece(pp);
+          if (!correctHash) {
+              if (listener != null)
+                  listener.storageChecked(this, piece, false);
+              return false;
+          }
 
-    // Early typecast, avoid possibly overflowing a temp integer
-    long start = (long) piece * (long) piece_size;
-    int i = 0;
-    long raflen = lengths[i];
-    while (start > raflen)
-      {
-        i++;
-        start -= raflen;
-        raflen = lengths[i];
-      }
+          // Early typecast, avoid possibly overflowing a temp integer
+          long start = (long) piece * (long) piece_size;
+          int i = 0;
+          long raflen = lengths[i];
+          while (start > raflen) {
+              i++;
+              start -= raflen;
+              raflen = lengths[i];
+          }
     
-    int written = 0;
-    int off = 0;
-    while (written < length)
-      {
-        int need = length - written;
-        int len = (start + need < raflen) ? need : (int)(raflen - start);
-        synchronized(RAFlock[i])
-          {
-            checkRAF(i);
-            rafs[i].seek(start);
-            rafs[i].write(bs, off + written, len);
-          }
-        written += len;
-        if (need - len > 0)
-          {
-            i++;
-            raflen = lengths[i];
-            start = 0;
+          int written = 0;
+          int off = 0;
+          int length = metainfo.getPieceLength(piece);
+          while (written < length) {
+              int need = length - written;
+              int len = (start + need < raflen) ? need : (int)(raflen - start);
+              synchronized(RAFlock[i]) {
+                  checkRAF(i);
+                  if (isSparse[i]) {
+                      // If the file is a newly created sparse file,
+                      // AND we aren't skipping it, balloon it with all
+                      // zeros to un-sparse it by allocating the space.
+                      // Obviously this could take a while.
+                      // Once we have written to it, it isn't empty/sparse any more.
+                      if (priorities == null || priorities[i] >= 0)
+                          balloonFile(i);
+                      else
+                          isSparse[i] = false;
+                  }
+                  rafs[i].seek(start);
+                  //rafs[i].write(bs, off + written, len);
+                  pp.write(rafs[i], off + written, len);
+              }
+              written += len;
+              if (need - len > 0) {
+                  i++;
+                  raflen = lengths[i];
+                  start = 0;
+              }
           }
+      } finally {
+          pp.release();
       }
 
     changed = true;
@@ -936,6 +1070,9 @@ public class Storage
             complete = needed == 0;
           }
       }
+    // tell listener after counts are updated
+    if (listener != null)
+        listener.storageChecked(this, piece, true);
 
     if (complete) {
       // do we also need to close all of the files and reopen
@@ -950,13 +1087,14 @@ public class Storage
       if (needed > 0) {
         if (listener != null)
             listener.setWantedPieces(this);
-        _util.debug("WARNING: Not really done, missing " + needed
-                    + " pieces", Snark.WARNING);
+        if (_log.shouldLog(Log.WARN))
+            _log.warn("WARNING: Not really done, missing " + needed
+                    + " pieces");
       }
     }
 
     return true;
-  }
+ }
 
   /**
    *  This is a dup of MetaInfo.getPieceLength() but we need it
@@ -1021,7 +1159,7 @@ public class Storage
   /**
    * Close unused RAFs - call periodically
    */
-  private static final long RAFCloseDelay = 7*60*1000;
+  private static final long RAFCloseDelay = 4*60*1000;
   public void cleanRAFs() {
     long cutoff = System.currentTimeMillis() - RAFCloseDelay;
     for (int i = 0; i < RAFlock.length; i++) {
@@ -1072,4 +1210,43 @@ public class Storage
     rafs[i] = null;
   }
 
+  /**
+   *  Create a metainfo.
+   *  Used in the installer build process; do not comment out.
+   *  @since 0.9.4
+   */
+  public static void main(String[] args) {
+      if (args.length < 1 || args.length > 2) {
+          System.err.println("Usage: Storage file-or-dir [announceURL]");
+          System.exit(1);
+      }
+      File base = new File(args[0]);
+      String announce = args.length == 2 ? args[1] : null;
+      I2PAppContext ctx = I2PAppContext.getGlobalContext();
+      I2PSnarkUtil util = new I2PSnarkUtil(ctx);
+      File file = null;
+      FileOutputStream out = null;
+      try {
+          Storage storage = new Storage(util, base, announce, null, false, null);
+          MetaInfo meta = storage.getMetaInfo();
+          file = new File(storage.getBaseName() + ".torrent");
+          out = new FileOutputStream(file);
+          out.write(meta.getTorrentData());
+          String hex = DataHelper.toString(meta.getInfoHash());
+          System.out.println("Created:     " + file);
+          System.out.println("InfoHash:    " + hex);
+          String basename = base.getName().replace(" ", "%20");
+          String magnet = MagnetURI.MAGNET_FULL + hex + "&dn=" + basename;
+          if (announce != null)
+              magnet += "&tr=" + announce;
+          System.out.println("Magnet:      " + magnet);
+      } catch (IOException ioe) {
+          if (file != null)
+              file.delete();
+          ioe.printStackTrace();
+          System.exit(1);
+      } finally {
+          try { if (out != null) out.close(); } catch (IOException ioe) {}
+      }
+  }
 }

apps/i2psnark/java/src/org/klomp/snark/StorageListener.java

@@ -23,7 +23,7 @@ package org.klomp.snark;
 /**
  * Callback used when Storage changes.
  */
-public interface StorageListener
+interface StorageListener
 {
   /**
    * Called when the storage creates a new file of a given length.
@@ -61,4 +61,6 @@ public interface StorageListener
    *
    */
   void setWantedPieces(Storage storage);
+
+  void addMessage(String message);
 }

apps/i2psnark/java/src/org/klomp/snark/Tracker.java

@@ -0,0 +1,28 @@
+/*
+ * Released into the public domain 
+ * with no warranty of any kind, either expressed or implied.  
+ */
+package org.klomp.snark;
+
+/**
+ * A structure for known trackers
+ *
+ * @since 0.9.1
+ */
+public class Tracker {
+
+    public final String name;
+    public final String announceURL;
+    public final String baseURL;
+    public final boolean supportsDetails;
+
+    /**
+     *  @param baseURL The web site, may be null
+     */
+    public Tracker(String name, String announceURL, String baseURL) {
+        this.name = name;
+        this.announceURL = announceURL;
+        this.baseURL = baseURL;
+        this.supportsDetails = name.contains("tracker2.postman.i2p");
+    }
+}