From ec51ea6513863460cbf920ffeb2b6a44cad4b5da Mon Sep 17 00:00:00 2001 From: zzz Date: Fri, 26 Nov 2010 00:32:00 +0000 Subject: [PATCH 01/10] * Console: Split initialNews.xml into a file for each language; don't copy to config dir at install. --- apps/routerconsole/java/build.xml | 5 + .../src/net/i2p/router/web/ContentHelper.java | 4 +- .../src/net/i2p/router/web/NewsHelper.java | 20 +++ apps/routerconsole/jsp/index.jsp | 3 +- build.xml | 11 +- history.txt | 9 ++ installer/resources/initialNews.xml | 127 ------------------ .../resources/initialNews/initialNews.xml | 20 +++ .../resources/initialNews/initialNews_br.xml | 21 +++ .../resources/initialNews/initialNews_de.xml | 21 +++ .../resources/initialNews/initialNews_es.xml | 21 +++ .../resources/initialNews/initialNews_nl.xml | 19 +++ .../resources/initialNews/initialNews_ru.xml | 16 +++ .../net/i2p/router/startup/WorkingDir.java | 3 +- 14 files changed, 163 insertions(+), 137 deletions(-) create mode 100644 apps/routerconsole/java/src/net/i2p/router/web/NewsHelper.java delete mode 100644 installer/resources/initialNews.xml create mode 100644 installer/resources/initialNews/initialNews.xml create mode 100644 installer/resources/initialNews/initialNews_br.xml create mode 100644 installer/resources/initialNews/initialNews_de.xml create mode 100644 installer/resources/initialNews/initialNews_es.xml create mode 100644 installer/resources/initialNews/initialNews_nl.xml create mode 100644 installer/resources/initialNews/initialNews_ru.xml diff --git a/apps/routerconsole/java/build.xml b/apps/routerconsole/java/build.xml index ae73734cd..22eadd52a 100644 --- a/apps/routerconsole/java/build.xml +++ b/apps/routerconsole/java/build.xml @@ -92,6 +92,11 @@ + + + + + - @@ -476,8 +474,12 @@ - - + + + + + + @@ -766,7 +768,6 @@ - diff --git a/history.txt b/history.txt index e283f8c0e..bdb01855d 100644 --- a/history.txt +++ b/history.txt @@ -1,3 +1,12 @@ +2010-11-27 zzz + * Console: Split initialNews.xml into a file for each language + don't copy to config dir at install. + * i2psnark: Clean up and enhance the PeerCoordinator's partial piece handling, + in preparation for more improvements + * LogManager: When not in router context, delay creating log file until required + * NetDb: Lower RouterInfo expiration time again + * Router: Prevent NCDFE after unzipping update file + 2010-11-25 dr|z3d * Console themes: Classic and Dark theme refresh. * Fix langbox overflow issue in all themes. diff --git a/installer/resources/initialNews.xml b/installer/resources/initialNews.xml deleted file mode 100644 index 15fc5088b..000000000 --- a/installer/resources/initialNews.xml +++ /dev/null @@ -1,127 +0,0 @@ - -
-

  • Congratulations on getting I2P installed!

-

-Welcome to I2P! -Please have patience as I2P boots up and finds peers. -

-

-While you are waiting, please adjust your bandwidth settings on the -configuration page. -

-

-Once you have a "shared clients" destination listed on the left, -please check out our -FAQ. -

-

-Point your IRC client to localhost:6668 and say hi to us on -#i2p-help or #i2p. -

-
- -
-

  • Wir gratulieren zur erfolgreichen Installation von I2P!

-

-Willkommen im I2P! -Hab noch etwas Geduld, während I2P startet und weitere I2P-Router findet. -

-

-Passe bitte in der Zwischenzeit deine Einstellungen zur Bandbreite auf der -Einstellungsseite an! -

-

-Sobald auf der linken Seite eine Verbindung namens "versch. Klienten" aufgelistet ist, kannst Du unsere FAQ besuchen. -

-

-Verbinde deinen IRC-Klienten mit dem Server auf localhost:6668 und schau bei uns im Kanal -#i2p-de, #i2p-help oder #i2p vorbei! -

-

-Wir wünschen Dir viel Freude an unserem Netz und hoffen, Deine Erwartungen zu erfüllen. -

-
- -
-

  • ¡Felicidades!, has instalado el enrutador I2P con éxito.

-

-¡Bienvenido a I2P!
-¡Ten todavía paciencia mientras I2P esté arrancando y encontrando otros enrutadores I2P! -

-

-Este es el momento ideal para adaptar tu configuración de ancho de banda en la -página de configuración. -

-

-En cuanto veas a la izquierda una conexión llamada "shared clients", puedes visitar nuestros FAQ. -

-

-¡Conécta tu cliente IRC con el servidor localhost:6668 y ven a saludarnos en los canales -#i2p-es, #i2p-help o #i2p! -

-

-Esperemos que tengas una buena experiencia con y en I2P. -

-
- -
-

  • Parabéns, você instalou o roteador I2P com êxito!

-

-Bem-vindo ao I2P!
-Seja paciente enquanto I2P ainda está iniciando-se e enquanto continuam sendo encontrados outros roteadores I2P! -

-

-Este é o momento ideal para personalizar a configuração de largura de banda na -página de configuração. -

-

-Quando você vê uma conexão no lado esquerdo chamada "shared clients", você pode visitar os nossos FAQ. -

-

-Conecte seu cliente de IRC para o servidor localhost:6668 e vem para nos cumprimentar aos canais -#i2p-br, #i2p-help ou #i2p! -

-

-Esperamos que você tenha uma boa experiência e I2P. -

-
- -
-

  • Gefeliciteerd met de installatie van I2P!

-

-Welkom bij I2P! -Heb wat geduld terwijl I2P opstart en peers zoekt. -

-

-Terwijl je wacht, pas je bandbreedte instellingen aan op de -configuratie pagina. -

-

-Wanneer je een "gedeelde clients" destination in de linker lijst hebt, -lees dan aub onze FAQ. -

-

-Verbind je IRC client met localhost:6668 en zeg Hallo in -#i2p-help of #i2p. -

-
- -
-

  • Поздравляем с успешным завершением установки I2P!

-

-Добро пожаловать в I2P! Немного терпения! I2P-маршрутизатору потребуется пара минут для запуска модулей и первого подключения к сети I2P. -

-

-Пока Вы ждете, самое время зайти в сетевые настройки и выставить ограничение скорости в соответствии со скоростью Вашего подключения к интернету. -

-

-Как только в панели слева в разделе «Локальные туннели» появится запись «коллективные клиенты» — I2P готов к работе. Подключайте Ваш IRC-клиент к серверу localhost:6668 и заходите сказать нам привет на канал -#i2p-help и #i2p. -

-

-Не забудьте заглянуть в наш FAQ. -

-
diff --git a/installer/resources/initialNews/initialNews.xml b/installer/resources/initialNews/initialNews.xml new file mode 100644 index 000000000..d69d6c644 --- /dev/null +++ b/installer/resources/initialNews/initialNews.xml @@ -0,0 +1,20 @@ +
+

  • Congratulations on getting I2P installed!

+

+Welcome to I2P! +Please have patience as I2P boots up and finds peers. +

+

+While you are waiting, please adjust your bandwidth settings on the +configuration page. +

+

+Once you have a "shared clients" destination listed on the left, +please check out our +FAQ. +

+

+Point your IRC client to localhost:6668 and say hi to us on +#i2p-help or #i2p. +

+
diff --git a/installer/resources/initialNews/initialNews_br.xml b/installer/resources/initialNews/initialNews_br.xml new file mode 100644 index 000000000..d3e1df50c --- /dev/null +++ b/installer/resources/initialNews/initialNews_br.xml @@ -0,0 +1,21 @@ +
+

  • Parabéns, você instalou o roteador I2P com êxito!

+

+Bem-vindo ao I2P!
+Seja paciente enquanto I2P ainda está iniciando-se e enquanto continuam sendo encontrados outros roteadores I2P! +

+

+Este é o momento ideal para personalizar a configuração de largura de banda na +página de configuração. +

+

+Quando você vê uma conexão no lado esquerdo chamada "shared clients", você pode visitar os nossos FAQ. +

+

+Conecte seu cliente de IRC para o servidor localhost:6668 e vem para nos cumprimentar aos canais +#i2p-br, #i2p-help ou #i2p! +

+

+Esperamos que você tenha uma boa experiência e I2P. +

+
diff --git a/installer/resources/initialNews/initialNews_de.xml b/installer/resources/initialNews/initialNews_de.xml new file mode 100644 index 000000000..38b037fc5 --- /dev/null +++ b/installer/resources/initialNews/initialNews_de.xml @@ -0,0 +1,21 @@ +
+

  • Wir gratulieren zur erfolgreichen Installation von I2P!

+

+Willkommen im I2P! +Hab noch etwas Geduld, während I2P startet und weitere I2P-Router findet. +

+

+Passe bitte in der Zwischenzeit deine Einstellungen zur Bandbreite auf der +Einstellungsseite an! +

+

+Sobald auf der linken Seite eine Verbindung namens "versch. Klienten" aufgelistet ist, kannst Du unsere FAQ besuchen. +

+

+Verbinde deinen IRC-Klienten mit dem Server auf localhost:6668 und schau bei uns im Kanal +#i2p-de, #i2p-help oder #i2p vorbei! +

+

+Wir wünschen Dir viel Freude an unserem Netz und hoffen, Deine Erwartungen zu erfüllen. +

+
diff --git a/installer/resources/initialNews/initialNews_es.xml b/installer/resources/initialNews/initialNews_es.xml new file mode 100644 index 000000000..eca5e4918 --- /dev/null +++ b/installer/resources/initialNews/initialNews_es.xml @@ -0,0 +1,21 @@ +
+

  • ¡Felicidades!, has instalado el enrutador I2P con éxito.

+

+¡Bienvenido a I2P!
+¡Ten todavía paciencia mientras I2P esté arrancando y encontrando otros enrutadores I2P! +

+

+Este es el momento ideal para adaptar tu configuración de ancho de banda en la +página de configuración. +

+

+En cuanto veas a la izquierda una conexión llamada "shared clients", puedes visitar nuestros FAQ. +

+

+¡Conécta tu cliente IRC con el servidor localhost:6668 y ven a saludarnos en los canales +#i2p-es, #i2p-help o #i2p! +

+

+Esperemos que tengas una buena experiencia con y en I2P. +

+
diff --git a/installer/resources/initialNews/initialNews_nl.xml b/installer/resources/initialNews/initialNews_nl.xml new file mode 100644 index 000000000..a968236d5 --- /dev/null +++ b/installer/resources/initialNews/initialNews_nl.xml @@ -0,0 +1,19 @@ +
+

  • Gefeliciteerd met de installatie van I2P!

+

+Welkom bij I2P! +Heb wat geduld terwijl I2P opstart en peers zoekt. +

+

+Terwijl je wacht, pas je bandbreedte instellingen aan op de +configuratie pagina. +

+

+Wanneer je een "gedeelde clients" destination in de linker lijst hebt, +lees dan aub onze FAQ. +

+

+Verbind je IRC client met localhost:6668 en zeg Hallo in +#i2p-help of #i2p. +

+
diff --git a/installer/resources/initialNews/initialNews_ru.xml b/installer/resources/initialNews/initialNews_ru.xml new file mode 100644 index 000000000..93eb2fd7a --- /dev/null +++ b/installer/resources/initialNews/initialNews_ru.xml @@ -0,0 +1,16 @@ +
+

  • Поздравляем с успешным завершением установки I2P!

+

+Добро пожаловать в I2P! Немного терпения! I2P-маршрутизатору потребуется пара минут для запуска модулей и первого подключения к сети I2P. +

+

+Пока Вы ждете, самое время зайти в сетевые настройки и выставить ограничение скорости в соответствии со скоростью Вашего подключения к интернету. +

+

+Как только в панели слева в разделе «Локальные туннели» появится запись «коллективные клиенты» — I2P готов к работе. Подключайте Ваш IRC-клиент к серверу localhost:6668 и заходите сказать нам привет на канал +#i2p-help и #i2p. +

+

+Не забудьте заглянуть в наш FAQ. +

+
diff --git a/router/java/src/net/i2p/router/startup/WorkingDir.java b/router/java/src/net/i2p/router/startup/WorkingDir.java index fd8efee3c..4fc466d27 100644 --- a/router/java/src/net/i2p/router/startup/WorkingDir.java +++ b/router/java/src/net/i2p/router/startup/WorkingDir.java @@ -149,7 +149,8 @@ public class WorkingDir { // this one must be after MIGRATE_BASE success &= migrateJettyXml(oldDirf, dirf); success &= migrateClientsConfig(oldDirf, dirf); - success &= copy(new File(oldDirf, "docs/news.xml"), new SecureDirectory(dirf, "docs")); + // for later news.xml updates (we don't copy initialNews.xml over anymore) + success &= (new SecureDirectory(dirf, "docs")) .mkdir(); // Report success or failure if (success) { From 9e250bc07db45fa1f9a421a613eea25a5964d907 Mon Sep 17 00:00:00 2001 From: zzz Date: Fri, 26 Nov 2010 00:32:44 +0000 Subject: [PATCH 02/10] fix hashcode and javadocs --- core/java/src/net/i2p/data/Payload.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/core/java/src/net/i2p/data/Payload.java b/core/java/src/net/i2p/data/Payload.java index 89cac8ff5..0371ed73f 100644 --- a/core/java/src/net/i2p/data/Payload.java +++ b/core/java/src/net/i2p/data/Payload.java @@ -19,6 +19,10 @@ import net.i2p.util.Log; * Defines the actual payload of a message being delivered, including the * standard encryption wrapping, as defined by the I2P data structure spec. * + * This is used mostly in I2CP, where we used to do end-to-end encryption. + * Since we don't any more, you probably just want to use the + * get/set EncryptedData methods. + * * @author jrandom */ public class Payload extends DataStructureImpl { @@ -32,6 +36,9 @@ public class Payload extends DataStructureImpl { /** * Retrieve the unencrypted body of the message. * + * Deprecated. + * Unless you are doing encryption, use getEncryptedData() instead. + * * @return body of the message, or null if the message has either not been * decrypted yet or if the hash is not correct */ @@ -43,15 +50,19 @@ public class Payload extends DataStructureImpl { * Populate the message body with data. This does not automatically encrypt * yet. * + * Deprecated. + * Unless you are doing encryption, use setEncryptedData() instead. */ public void setUnencryptedData(byte[] data) { _unencryptedData = data; } + /** the real data */ public byte[] getEncryptedData() { return _encryptedData; } + /** the real data */ public void setEncryptedData(byte[] data) { _encryptedData = data; } @@ -100,7 +111,7 @@ public class Payload extends DataStructureImpl { @Override public int hashCode() { - return DataHelper.hashCode(_unencryptedData); + return DataHelper.hashCode(_encryptedData != null ? _encryptedData : _unencryptedData); } @Override From 1197a5c8c903fd2172d167fd870d9597477ac12c Mon Sep 17 00:00:00 2001 From: zzz Date: Fri, 26 Nov 2010 00:33:40 +0000 Subject: [PATCH 03/10] reduce RouterInfo expiration again --- .../networkdb/kademlia/KademliaNetworkDatabaseFacade.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/router/java/src/net/i2p/router/networkdb/kademlia/KademliaNetworkDatabaseFacade.java b/router/java/src/net/i2p/router/networkdb/kademlia/KademliaNetworkDatabaseFacade.java index acede96ed..0ae718104 100644 --- a/router/java/src/net/i2p/router/networkdb/kademlia/KademliaNetworkDatabaseFacade.java +++ b/router/java/src/net/i2p/router/networkdb/kademlia/KademliaNetworkDatabaseFacade.java @@ -686,6 +686,8 @@ public class KademliaNetworkDatabaseFacade extends NetworkDatabaseFacade { return rv; } + private static final int MIN_ROUTERS = 90; + /** * Determine whether this routerInfo will be accepted as valid and current * given what we know now. @@ -694,9 +696,9 @@ public class KademliaNetworkDatabaseFacade extends NetworkDatabaseFacade { String validate(Hash key, RouterInfo routerInfo) throws IllegalArgumentException { long now = _context.clock().now(); boolean upLongEnough = _context.router().getUptime() > 60*60*1000; - // Once we're over 120 routers, reduce the expiration time down from the default, + // Once we're over MIN_ROUTERS routers, reduce the expiration time down from the default, // as a crude way of limiting memory usage. - // i.e. at 300 routers the expiration time will be about half the default, etc. + // i.e. at 2*MIN_ROUTERS routers the expiration time will be about half the default, etc. // And if we're floodfill, we can keep the expiration really short, since // we are always getting the latest published to us. // As the net grows this won't be sufficient, and we'll have to implement @@ -708,7 +710,7 @@ public class KademliaNetworkDatabaseFacade extends NetworkDatabaseFacade { // _kb.size() includes leasesets but that's ok adjustedExpiration = Math.min(ROUTER_INFO_EXPIRATION, ROUTER_INFO_EXPIRATION_MIN + - ((ROUTER_INFO_EXPIRATION - ROUTER_INFO_EXPIRATION_MIN) * 120 / (_kb.size() + 1))); + ((ROUTER_INFO_EXPIRATION - ROUTER_INFO_EXPIRATION_MIN) * MIN_ROUTERS / (_kb.size() + 1))); if (!key.equals(routerInfo.getIdentity().getHash())) { if (_log.shouldLog(Log.WARN)) From c3a2982154ced973fe559b71beb8295916cb1dd5 Mon Sep 17 00:00:00 2001 From: zzz Date: Fri, 26 Nov 2010 00:36:07 +0000 Subject: [PATCH 04/10] * Router: Prevent NCDFE after unzipping update file --- router/java/src/net/i2p/router/Router.java | 92 ++++++++++++++-------- 1 file changed, 60 insertions(+), 32 deletions(-) diff --git a/router/java/src/net/i2p/router/Router.java b/router/java/src/net/i2p/router/Router.java index eede0c768..459be0ec7 100644 --- a/router/java/src/net/i2p/router/Router.java +++ b/router/java/src/net/i2p/router/Router.java @@ -191,7 +191,9 @@ public class Router { // This is here so that we can get the directory location from the context // for the ping file - if (!beginMarkingLiveliness()) { + // Check for other router but do not start a thread yet so the update doesn't cause + // a NCDFE + if (!isOnlyRouterRunning()) { System.err.println("ERROR: There appears to be another router already running!"); System.err.println(" Please make sure to shut down old instances before starting up"); System.err.println(" a new one. If you are positive that no other instance is running,"); @@ -215,6 +217,11 @@ public class Router { // overwrite an existing running router's jar files. Other than ours. installUpdates(); + // ********* Start no threads before here ********* // + // + // NOW we can start the ping file thread. + beginMarkingLiveliness(); + // Apps may use this as an easy way to determine if they are in the router JVM // But context.isRouterContext() is even easier... // Both of these as of 0.7.9 @@ -1163,38 +1170,50 @@ public class Router { // verify the whole thing first // we could remember this fails, and not bother restarting, but who cares... boolean ok = FileUtil.verifyZip(updateFile); - if (ok) - ok = FileUtil.extractZip(updateFile, _context.getBaseDir()); - if (ok) - System.out.println("INFO: Update installed"); - else - System.out.println("ERROR: Update failed!"); - if (!ok) { - // we can't leave the file in place or we'll continually restart, so rename it - File bad = new File(_context.getRouterDir(), "BAD-" + UPDATE_FILE); - boolean renamed = updateFile.renameTo(bad); - if (renamed) { - System.out.println("Moved update file to " + bad.getAbsolutePath()); - } else { - System.out.println("Deleting file " + updateFile.getAbsolutePath()); - ok = true; // so it will be deleted - } - } if (ok) { // This may be useful someday. First added in 0.8.2 + // Moved above the extract so we don't NCDFE _config.put("router.updateLastInstalled", "" + System.currentTimeMillis()); saveConfig(); - boolean deleted = updateFile.delete(); - if (!deleted) { - System.out.println("ERROR: Unable to delete the update file!"); - updateFile.deleteOnExit(); - } + ok = FileUtil.extractZip(updateFile, _context.getBaseDir()); + } + + // Very important - we have now trashed our jars. + // After this point, do not use any new I2P classes, or they will fail to load + // and we will die with NCDFE. + // Ideally, do not use I2P classes at all, new or not. + try { + if (ok) + System.out.println("INFO: Update installed"); + else + System.out.println("ERROR: Update failed!"); + if (!ok) { + // we can't leave the file in place or we'll continually restart, so rename it + File bad = new File(_context.getRouterDir(), "BAD-" + UPDATE_FILE); + boolean renamed = updateFile.renameTo(bad); + if (renamed) { + System.out.println("Moved update file to " + bad.getAbsolutePath()); + } else { + System.out.println("Deleting file " + updateFile.getAbsolutePath()); + ok = true; // so it will be deleted + } + } + if (ok) { + boolean deleted = updateFile.delete(); + if (!deleted) { + System.out.println("ERROR: Unable to delete the update file!"); + updateFile.deleteOnExit(); + } + } + // exit whether ok or not + if (System.getProperty("wrapper.version") != null) + System.out.println("INFO: Restarting after update"); + else + System.out.println("WARNING: Exiting after update, restart I2P"); + } catch (Throwable t) { + // hide the NCDFE + // hopefully the update file got deleted or we will loop } - // exit whether ok or not - if (System.getProperty("wrapper.version") != null) - System.out.println("INFO: Restarting after update"); - else - System.out.println("WARNING: Exiting after update, restart I2P"); System.exit(EXIT_HARD_RESTART); } } @@ -1230,13 +1249,14 @@ public class Router { static final long LIVELINESS_DELAY = 60*1000; /** - * Start a thread that will periodically update the file "router.ping", but if + * Check the file "router.ping", but if * that file already exists and was recently written to, return false as there is - * another instance running + * another instance running. * * @return true if the router is the only one running + * @since 0.8.2 */ - private boolean beginMarkingLiveliness() { + private boolean isOnlyRouterRunning() { File f = getPingFile(); if (f.exists()) { long lastWritten = f.lastModified(); @@ -1247,12 +1267,20 @@ public class Router { return false; } } + return true; + } + + /** + * Start a thread that will periodically update the file "router.ping". + * isOnlyRouterRunning() MUST have been called previously. + */ + private void beginMarkingLiveliness() { + File f = getPingFile(); // not an I2PThread for context creation issues Thread t = new Thread(new MarkLiveliness(_context, this, f)); t.setName("Mark router liveliness"); t.setDaemon(true); t.start(); - return true; } public static final String PROP_BANDWIDTH_SHARE_PERCENTAGE = "router.sharePercentage"; From 358846ab046e1c170f840f30f3f2604eb7a47801 Mon Sep 17 00:00:00 2001 From: zzz Date: Fri, 26 Nov 2010 00:37:20 +0000 Subject: [PATCH 05/10] * LogManager: When not in router context, delay creating log file until required --- core/java/src/net/i2p/util/LogManager.java | 51 ++++++++++++++-------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/core/java/src/net/i2p/util/LogManager.java b/core/java/src/net/i2p/util/LogManager.java index 634b4c1ad..6c9a062e4 100644 --- a/core/java/src/net/i2p/util/LogManager.java +++ b/core/java/src/net/i2p/util/LogManager.java @@ -66,8 +66,8 @@ public class LogManager { public final static String DEFAULT_DEFAULTLEVEL = Log.STR_ERROR; public final static String DEFAULT_ONSCREENLEVEL = Log.STR_CRIT; - private I2PAppContext _context; - private Log _log; + private final I2PAppContext _context; + private final Log _log; /** when was the config file last read (or -1 if never) */ private long _configLastRead; @@ -75,11 +75,11 @@ public class LogManager { /** the config file */ private File _locationFile; /** Ordered list of LogRecord elements that have not been written out yet */ - private LinkedBlockingQueue _records; + private final LinkedBlockingQueue _records; /** List of explicit overrides of log levels (LogLimit objects) */ - private Set _limits; + private final Set _limits; /** String (scope) or Log.LogScope to Log object */ - private ConcurrentHashMap _logs; + private final ConcurrentHashMap _logs; /** who clears and writes our records */ private LogWriter _writer; @@ -108,7 +108,7 @@ public class LogManager { /** how many records we want to buffer in the "recent logs" list */ private int _consoleBufferSize; /** the actual "recent logs" list */ - private LogConsoleBuffer _consoleBuffer; + private final LogConsoleBuffer _consoleBuffer; private boolean _alreadyNoticedMissingConfig; @@ -119,17 +119,17 @@ public class LogManager { _limits = new ConcurrentHashSet(); _logs = new ConcurrentHashMap(128); _defaultLimit = Log.ERROR; - _configLastRead = 0; _context = context; _log = getLog(LogManager.class); String location = context.getProperty(CONFIG_LOCATION_PROP, CONFIG_LOCATION_DEFAULT); setConfig(location); _consoleBuffer = new LogConsoleBuffer(context); - _writer = new LogWriter(this); - Thread t = new I2PThread(_writer); - t.setName("LogWriter"); - t.setDaemon(true); - t.start(); + // If we aren't in the router context, delay creating the LogWriter until required, + // so it doesn't create a log directory and log files unless there is output. + // In the router context, we have to rotate to a new log file at startup or the logs.jsp + // page will display the old log. + if (context.isRouterContext()) + startLogWriter(); try { Runtime.getRuntime().addShutdownHook(new ShutdownHook()); } catch (IllegalStateException ise) { @@ -138,9 +138,16 @@ public class LogManager { //System.out.println("Created logManager " + this + " with context: " + context); } - private LogManager() { // nop + /** @since 0.8.2 */ + private synchronized void startLogWriter() { + // yeah, this doesn't always work, _writer should be volatile + if (_writer != null) + return; + _writer = new LogWriter(this); + Thread t = new I2PThread(_writer, "LogWriter", true); + t.start(); } - + public Log getLog(Class cls) { return getLog(cls, null); } public Log getLog(String name) { return getLog(null, name); } public Log getLog(Class cls, String name) { @@ -169,6 +176,7 @@ public class LogManager { public LogConsoleBuffer getBuffer() { return _consoleBuffer; } + /** @deprecated unused */ public void setDisplayOnScreen(boolean yes) { _displayOnScreen = yes; } @@ -181,6 +189,7 @@ public class LogManager { return _onScreenLimit; } + /** @deprecated unused */ public void setDisplayOnScreenLevel(int level) { _onScreenLimit = level; } @@ -189,6 +198,7 @@ public class LogManager { return _consoleBufferSize; } + /** @deprecated unused */ public void setConsoleBufferSize(int numRecords) { _consoleBufferSize = numRecords; } @@ -201,6 +211,8 @@ public class LogManager { } public String currentFile() { + if (_writer == null) + return ("No log file created yet"); return _writer.currentFile(); } @@ -209,6 +221,9 @@ public class LogManager { * */ void addRecord(LogRecord record) { + if ((!_context.isRouterContext()) && _writer == null) + startLogWriter(); + _records.offer(record); int numRecords = _records.size(); @@ -617,9 +632,11 @@ public class LogManager { } public void shutdown() { - _log.log(Log.WARN, "Shutting down logger"); - _writer.flushRecords(false); - _writer.stopWriting(); + if (_writer != null) { + _log.log(Log.WARN, "Shutting down logger"); + _writer.flushRecords(false); + _writer.stopWriting(); + } } private static int __id = 0; From d37944e081581085cf5668aa4f0bb428104fae8c Mon Sep 17 00:00:00 2001 From: zzz Date: Fri, 26 Nov 2010 00:41:20 +0000 Subject: [PATCH 06/10] javadoc --- core/java/src/net/i2p/data/PrivateKeyFile.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/java/src/net/i2p/data/PrivateKeyFile.java b/core/java/src/net/i2p/data/PrivateKeyFile.java index eab03576d..0c752f9d6 100644 --- a/core/java/src/net/i2p/data/PrivateKeyFile.java +++ b/core/java/src/net/i2p/data/PrivateKeyFile.java @@ -23,6 +23,7 @@ import net.i2p.crypto.DSAEngine; * This helper class reads and writes files in the * same "eepPriv.dat" format used by the client code. * The format is: + *
  *  - Destination (387 bytes if no certificate, otherwise longer)
  *     - Public key (256 bytes)
  *     - Signing Public key (128 bytes)
@@ -32,6 +33,7 @@ import net.i2p.crypto.DSAEngine;
  *  - Private key (256 bytes)
  *  - Signing Private key (20 bytes)
  * Total 663 bytes
+ *
* * @author welterde, zzz */ From 7f1ace4dbed7eb30c4071052d080f7298fac372b Mon Sep 17 00:00:00 2001 From: zzz Date: Fri, 26 Nov 2010 00:44:00 +0000 Subject: [PATCH 07/10] * i2psnark: Clean up and enhance the PeerCoordinator's partial piece handling, in preparation for more improvements --- .../src/org/klomp/snark/PartialPiece.java | 102 ++++++++++ .../java/src/org/klomp/snark/Peer.java | 8 +- .../src/org/klomp/snark/PeerCoordinator.java | 191 +++++++++++------- .../src/org/klomp/snark/PeerListener.java | 15 +- .../java/src/org/klomp/snark/PeerState.java | 125 ++++++------ .../java/src/org/klomp/snark/Piece.java | 5 +- .../java/src/org/klomp/snark/Request.java | 1 + 7 files changed, 304 insertions(+), 143 deletions(-) create mode 100644 apps/i2psnark/java/src/org/klomp/snark/PartialPiece.java diff --git a/apps/i2psnark/java/src/org/klomp/snark/PartialPiece.java b/apps/i2psnark/java/src/org/klomp/snark/PartialPiece.java new file mode 100644 index 000000000..8ecae5778 --- /dev/null +++ b/apps/i2psnark/java/src/org/klomp/snark/PartialPiece.java @@ -0,0 +1,102 @@ +package org.klomp.snark; + +/** + * This is the class passed from PeerCoordinator to PeerState so + * PeerState may start requests. + * + * It is also passed from PeerState to PeerCoordinator when + * a piece is not completely downloaded, for example + * when the Peer disconnects or chokes. + */ +class PartialPiece implements Comparable { + + private final int piece; + private final byte[] bs; + private final int off; + private final long createdTime; + + /** + * Used by PeerCoordinator. + * Creates a new PartialPiece, with no chunks yet downloaded. + * Allocates the data. + * + * @param piece Piece number requested. + * @param bs length must be equal to the piece length + */ + public PartialPiece (int piece, int len) throws OutOfMemoryError { + this.piece = piece; + this.bs = new byte[len]; + this.off = 0; + this.createdTime = 0; + } + + /** + * Used by PeerState. + * Creates a new PartialPiece, with chunks up to but not including + * firstOutstandingRequest already downloaded and stored in the Request byte array. + * + * Note that this cannot handle gaps; chunks after a missing chunk cannot be saved. + * That would be harder. + * + * @param firstOutstandingRequest the first request not fulfilled for the piece + */ + public PartialPiece (Request firstOutstandingRequest) { + this.piece = firstOutstandingRequest.piece; + this.bs = firstOutstandingRequest.bs; + this.off = firstOutstandingRequest.off; + this.createdTime = System.currentTimeMillis(); + } + + /** + * Convert this PartialPiece to a request for the next chunk. + * Used by PeerState only. + */ + + public Request getRequest() { + return new Request(this.piece, this.bs, this.off, Math.min(this.bs.length - this.off, PeerState.PARTSIZE)); + } + + /** piece number */ + public int getPiece() { + return this.piece; + } + + /** how many bytes are good */ + public int getDownloaded() { + return this.off; + } + + public long getCreated() { + return this.createdTime; + } + + /** + * Highest downloaded first + */ + public int compareTo(Object o) throws ClassCastException { + return ((PartialPiece)o).off - this.off; // reverse + } + + @Override + public int hashCode() { + return piece * 7777; + } + + /** + * Make this simple so PeerCoordinator can keep a List. + * Warning - compares piece number only! + */ + @Override + public boolean equals(Object o) { + if (o instanceof PartialPiece) { + PartialPiece pp = (PartialPiece)o; + return pp.piece == this.piece; + } + return false; + } + + @Override + public String toString() { + return "Partial(" + piece + ',' + off + ',' + bs.length + ')'; + } +} diff --git a/apps/i2psnark/java/src/org/klomp/snark/Peer.java b/apps/i2psnark/java/src/org/klomp/snark/Peer.java index e747332d8..257c5ff9a 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Peer.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Peer.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.Arrays; +import java.util.List; import net.i2p.client.streaming.I2PSocket; import net.i2p.util.Log; @@ -368,8 +369,11 @@ public class Peer implements Comparable if (this.deregister) { PeerListener p = s.listener; if (p != null) { - p.savePeerPartial(s); - p.markUnrequested(this); + List pcs = s.returnPartialPieces(); + if (!pcs.isEmpty()) + p.savePartialPieces(this, pcs); + // now covered by savePartialPieces + //p.markUnrequested(this); } } state = null; diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java index c8f8d7d7f..410ac43c5 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java @@ -74,6 +74,9 @@ public class PeerCoordinator implements PeerListener // Some random wanted pieces private List wantedPieces; + /** partial pieces */ + private final List partialPieces; + private boolean halted = false; private final CoordinatorListener listener; @@ -94,6 +97,7 @@ public class PeerCoordinator implements PeerListener this.snark = torrent; setWantedPieces(); + partialPieces = new ArrayList(getMaxConnections() + 1); // Install a timer to check the uploaders. // Randomize the first start time so multiple tasks are spread out, @@ -293,7 +297,9 @@ public class PeerCoordinator implements PeerListener removePeerFromPieces(peer); } // delete any saved orphan partial piece - savedRequest = null; + synchronized (partialPieces) { + partialPieces.clear(); + } } public void connected(Peer peer) @@ -773,6 +779,9 @@ public class PeerCoordinator implements PeerListener wantedPieces.remove(p); } + // just in case + removePartialPiece(piece); + // Announce to the world we have it! // Disconnect from other seeders when we get the last piece synchronized(peers) @@ -866,70 +875,123 @@ public class PeerCoordinator implements PeerListener } } - - /** Simple method to save a partial piece on peer disconnection + /** + * Save partial pieces on peer disconnection * and hopefully restart it later. - * Only one partial piece is saved at a time. - * Replace it if a new one is bigger or the old one is too old. + * Replace a partial piece in the List if the new one is bigger. * Storage method is private so we can expand to save multiple partials * if we wish. + * + * Also mark the piece unrequested if this peer was the only one. + * + * @param peer partials, must include the zero-offset (empty) ones too + * @since 0.8.2 */ - private Request savedRequest = null; - private long savedRequestTime = 0; - public void savePeerPartial(PeerState state) + public void savePartialPieces(Peer peer, List partials) { - if (halted) - return; - Request req = state.getPartialRequest(); - if (req == null) - return; - if (savedRequest == null || - req.off > savedRequest.off || - System.currentTimeMillis() > savedRequestTime + (15 * 60 * 1000)) { - if (savedRequest == null || (req.piece != savedRequest.piece && req.off != savedRequest.off)) { - if (_log.shouldLog(Log.DEBUG)) { - _log.debug(" Saving orphaned partial piece " + req); - if (savedRequest != null) - _log.debug(" (Discarding previously saved orphan) " + savedRequest); - } + if (halted) + return; + if (_log.shouldLog(Log.INFO)) + _log.info("Partials received from " + peer + ": " + partials); + synchronized(partialPieces) { + for (PartialPiece pp : partials) { + if (pp.getDownloaded() > 0) { + // PartialPiece.equals() only compares piece number, which is what we want + int idx = partialPieces.indexOf(pp); + if (idx < 0) { + partialPieces.add(pp); + if (_log.shouldLog(Log.INFO)) + _log.info("Saving orphaned partial piece (new) " + pp); + } else if (idx >= 0 && pp.getDownloaded() > partialPieces.get(idx).getDownloaded()) { + // replace what's there now + partialPieces.set(idx, pp); + if (_log.shouldLog(Log.INFO)) + _log.info("Saving orphaned partial piece (bigger) " + pp); + } else { + if (_log.shouldLog(Log.INFO)) + _log.info("Discarding partial piece (not bigger)" + pp); + } + int max = getMaxConnections(); + if (partialPieces.size() > max) { + // sorts by remaining bytes, least first + Collections.sort(partialPieces); + PartialPiece gone = partialPieces.remove(max); + if (_log.shouldLog(Log.INFO)) + _log.info("Discarding orphaned partial piece (list full)" + gone); + } + } // else drop the empty partial piece + // synchs on wantedPieces... + markUnrequestedIfOnlyOne(peer, pp.getPiece()); + } + if (_log.shouldLog(Log.INFO)) + _log.info("Partial list size now: " + partialPieces.size()); } - savedRequest = req; - savedRequestTime = System.currentTimeMillis(); - } else { - if (req.piece != savedRequest.piece) - if (_log.shouldLog(Log.DEBUG)) - _log.debug(" Discarding orphaned partial piece " + req); - } } - /** Return partial piece if it's still wanted and peer has it. + /** + * Return partial piece to the PeerState if it's still wanted and peer has it. + * @param havePieces pieces the peer has, the rv will be one of these + * + * @return PartialPiece or null + * @since 0.8.2 */ - public Request getPeerPartial(BitField havePieces) { - if (savedRequest == null) - return null; - if (! havePieces.get(savedRequest.piece)) { - if (_log.shouldLog(Log.DEBUG)) - _log.debug("Peer doesn't have orphaned piece " + savedRequest); - return null; - } - synchronized(wantedPieces) - { - for(Iterator iter = wantedPieces.iterator(); iter.hasNext(); ) { - Piece piece = iter.next(); - if (piece.getId() == savedRequest.piece) { - Request req = savedRequest; - piece.setRequested(true); - if (_log.shouldLog(Log.DEBUG)) - _log.debug("Restoring orphaned partial piece " + req); - savedRequest = null; - return req; + public PartialPiece getPartialPiece(Peer peer, BitField havePieces) { + // do it in this order to avoid deadlock (same order as in savePartialPieces()) + synchronized(partialPieces) { + synchronized(wantedPieces) { + // sorts by remaining bytes, least first + Collections.sort(partialPieces); + for (Iterator iter = partialPieces.iterator(); iter.hasNext(); ) { + PartialPiece pp = iter.next(); + int savedPiece = pp.getPiece(); + if (havePieces.get(savedPiece)) { + // this is just a double-check, it should be in there + for(Piece piece : wantedPieces) { + if (piece.getId() == savedPiece) { + piece.setRequested(true); + iter.remove(); + if (_log.shouldLog(Log.INFO)) { + _log.info("Restoring orphaned partial piece " + pp + + " Partial list size now: " + partialPieces.size()); + } + return pp; + } + } + } + } + } + } + // ...and this section turns this into the general move-requests-around code! + // Temporary? So PeerState never calls wantPiece() directly for now... + int piece = wantPiece(peer, havePieces); + if (piece >= 0) { + try { + return new PartialPiece(piece, metainfo.getPieceLength(piece)); + } catch (OutOfMemoryError oom) { + if (_log.shouldLog(Log.WARN)) + _log.warn("OOM creating new partial piece"); + } + } + if (_log.shouldLog(Log.DEBUG)) + _log.debug("We have no partial piece to return"); + return null; + } + + /** + * Remove saved state for this piece. + * Unless we are in the end game there shouldnt be anything in there. + * Do not call with wantedPieces lock held (deadlock) + */ + private void removePartialPiece(int piece) { + synchronized(partialPieces) { + for (Iterator iter = partialPieces.iterator(); iter.hasNext(); ) { + PartialPiece pp = iter.next(); + if (pp.getPiece() == piece) { + iter.remove(); + // there should be only one but keep going to be sure + } } - } } - if (_log.shouldLog(Log.DEBUG)) - _log.debug("We no longer want orphaned piece " + savedRequest); - savedRequest = null; - return null; } /** Clear the requested flag for a piece if the peer @@ -947,13 +1009,12 @@ public class PeerCoordinator implements PeerListener continue; if (p.state == null) continue; - int[] arr = p.state.getRequestedPieces(); - for (int i = 0; arr[i] >= 0; i++) - if(arr[i] == piece) { + // FIXME don't go into the state + if (p.state.getRequestedPieces().contains(Integer.valueOf(piece))) { if (_log.shouldLog(Log.DEBUG)) _log.debug("Another peer is requesting piece " + piece); return; - } + } } } @@ -973,20 +1034,6 @@ public class PeerCoordinator implements PeerListener } } - /** Mark a peer's requested pieces unrequested when it is disconnected - ** Once for each piece - ** This is enough trouble, maybe would be easier just to regenerate - ** the requested list from scratch instead. - */ - public void markUnrequested(Peer peer) - { - if (halted || peer.state == null) - return; - int[] arr = peer.state.getRequestedPieces(); - for (int i = 0; arr[i] >= 0; i++) - markUnrequestedIfOnlyOne(peer, arr[i]); - } - /** Return number of allowed uploaders for this torrent. ** Check with Snark to see if we are over the total upload limit. */ diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerListener.java b/apps/i2psnark/java/src/org/klomp/snark/PeerListener.java index 2cbd34bb5..30f6fe453 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerListener.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerListener.java @@ -20,10 +20,12 @@ package org.klomp.snark; +import java.util.List; + /** * Listener for Peer events. */ -public interface PeerListener +interface PeerListener { /** * Called when the connection to the peer has started and the @@ -151,7 +153,7 @@ public interface PeerListener * * @param state the PeerState for the peer */ - void savePeerPartial(PeerState state); /* FIXME Exporting non-public type through public API FIXME */ + void savePartialPieces(Peer peer, List pcs); /** * Called when a peer has connected and there may be a partially @@ -161,12 +163,5 @@ public interface PeerListener * * @return request (contains the partial data and valid length) */ - Request getPeerPartial(BitField havePieces); /* FIXME Exporting non-public type through public API FIXME */ - - /** Mark a peer's requested pieces unrequested when it is disconnected - * This prevents premature end game - * - * @param peer the peer that is disconnecting - */ - void markUnrequested(Peer peer); + PartialPiece getPartialPiece(Peer peer, BitField havePieces); } diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerState.java b/apps/i2psnark/java/src/org/klomp/snark/PeerState.java index d649b8227..3a8487c6d 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerState.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerState.java @@ -23,9 +23,11 @@ package org.klomp.snark; import java.io.ByteArrayInputStream; import java.io.InputStream; import java.util.ArrayList; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import net.i2p.I2PAppContext; import net.i2p.util.Log; @@ -36,9 +38,9 @@ import org.klomp.snark.bencode.BEValue; class PeerState implements DataLoader { private final Log _log = I2PAppContext.getGlobalContext().logManager().getLog(PeerState.class); - final Peer peer; + private final Peer peer; final PeerListener listener; - final MetaInfo metainfo; + private final MetaInfo metainfo; // Interesting and choking describes whether we are interested in or // are choking the other side. @@ -54,6 +56,7 @@ class PeerState implements DataLoader long downloaded; long uploaded; + /** the pieces the peer has */ BitField bitfield; // Package local for use by Peer. @@ -102,6 +105,12 @@ class PeerState implements DataLoader if (interesting && !choked) request(resend); + + if (choked) { + // TODO + // savePartialPieces + // clear request list + } } void interestedMessage(boolean interest) @@ -308,8 +317,11 @@ class PeerState implements DataLoader } } + /** + * @return index in outstandingRequests or -1 + */ synchronized private int getFirstOutstandingRequest(int piece) - { + { for (int i = 0; i < outstandingRequests.size(); i++) if (outstandingRequests.get(i).piece == piece) return i; @@ -397,54 +409,56 @@ class PeerState implements DataLoader } - // get longest partial piece - synchronized Request getPartialRequest() - { - Request req = null; - for (int i = 0; i < outstandingRequests.size(); i++) { - Request r1 = outstandingRequests.get(i); - int j = getFirstOutstandingRequest(r1.piece); - if (j == -1) - continue; - Request r2 = outstandingRequests.get(j); - if (r2.off > 0 && ((req == null) || (r2.off > req.off))) - req = r2; - } - if (pendingRequest != null && req != null && pendingRequest.off < req.off) { - if (pendingRequest.off != 0) - req = pendingRequest; - else - req = null; - } - return req; + /** + * @return lowest offset of any request for the piece + * @since 0.8.2 + */ + synchronized private Request getLowestOutstandingRequest(int piece) { + Request rv = null; + int lowest = Integer.MAX_VALUE; + for (Request r : outstandingRequests) { + if (r.piece == piece && r.off < lowest) { + lowest = r.off; + rv = r; + } + } + if (pendingRequest != null && + pendingRequest.piece == piece && pendingRequest.off < lowest) + rv = pendingRequest; + + if (_log.shouldLog(Log.DEBUG)) + _log.debug(peer + " lowest for " + piece + " is " + rv + " out of " + pendingRequest + " and " + outstandingRequests); + return rv; } /** - * return array of pieces terminated by -1 - * remove most duplicates - * but still could be some duplicates, not guaranteed - * TODO rework this Java-style to return a Set or a List + * get partial pieces, give them back to PeerCoordinator + * @return List of PartialPieces, even those with an offset == 0, or empty list + * @since 0.8.2 */ - synchronized int[] getRequestedPieces() + synchronized List returnPartialPieces() { - int size = outstandingRequests.size(); - int[] arr = new int[size+2]; - int pc = -1; - int pos = 0; - if (pendingRequest != null) { - pc = pendingRequest.piece; - arr[pos++] = pc; - } - Request req = null; - for (int i = 0; i < size; i++) { - Request r1 = outstandingRequests.get(i); - if (pc != r1.piece) { - pc = r1.piece; - arr[pos++] = pc; + Set pcs = getRequestedPieces(); + List rv = new ArrayList(pcs.size()); + for (Integer p : pcs) { + Request req = getLowestOutstandingRequest(p.intValue()); + if (req != null) + rv.add(new PartialPiece(req)); } - } - arr[pos] = -1; - return(arr); + return rv; + } + + /** + * @return all pieces we are currently requesting, or empty Set + */ + synchronized Set getRequestedPieces() { + Set rv = new HashSet(outstandingRequests.size() + 1); + for (Request req : outstandingRequests) { + rv.add(Integer.valueOf(req.piece)); + if (pendingRequest != null) + rv.add(Integer.valueOf(pendingRequest.piece)); + } + return rv; } void cancelMessage(int piece, int begin, int length) @@ -555,6 +569,8 @@ class PeerState implements DataLoader { synchronized (this) { out.sendRequests(outstandingRequests); + if (_log.shouldLog(Log.DEBUG)) + _log.debug("Resending requests to " + peer + outstandingRequests); } } @@ -620,24 +636,17 @@ class PeerState implements DataLoader if (bitfield != null) { // Check for adopting an orphaned partial piece - Request r = listener.getPeerPartial(bitfield); - if (r != null) { - // Check that r not already in outstandingRequests - int[] arr = getRequestedPieces(); - boolean found = false; - for (int i = 0; arr[i] >= 0; i++) { - if (arr[i] == r.piece) { - found = true; - break; - } - } - if (!found) { + PartialPiece pp = listener.getPartialPiece(peer, bitfield); + if (pp != null) { + // Double-check that r not already in outstandingRequests + if (!getRequestedPieces().contains(Integer.valueOf(pp.getPiece()))) { + Request r = pp.getRequest(); outstandingRequests.add(r); if (!choked) out.sendRequest(r); lastRequest = r; return true; - } + } } // Note that in addition to the bitfield, PeerCoordinator uses diff --git a/apps/i2psnark/java/src/org/klomp/snark/Piece.java b/apps/i2psnark/java/src/org/klomp/snark/Piece.java index 0ae9570e1..68b2ddfd4 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Piece.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Piece.java @@ -5,7 +5,10 @@ import java.util.Set; import net.i2p.util.ConcurrentHashSet; -public class Piece implements Comparable { +/** + * This class is used solely by PeerCoordinator. + */ +class Piece implements Comparable { private int id; private Set peers; diff --git a/apps/i2psnark/java/src/org/klomp/snark/Request.java b/apps/i2psnark/java/src/org/klomp/snark/Request.java index cc8600b13..6c086ebae 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Request.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Request.java @@ -22,6 +22,7 @@ package org.klomp.snark; /** * Holds all information needed for a partial piece request. + * This class should be used only by PeerState, PeerConnectionIn, and PeerConnectionOut. */ class Request { From e1e6db2b3c1c8a005f517a1bb105c328e257fafd Mon Sep 17 00:00:00 2001 From: zzz Date: Fri, 26 Nov 2010 14:04:49 +0000 Subject: [PATCH 08/10] Round NTCP timestamps to reduce clock bias --- .../src/net/i2p/router/transport/ntcp/EstablishState.java | 4 ++-- .../src/net/i2p/router/transport/ntcp/NTCPConnection.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/router/java/src/net/i2p/router/transport/ntcp/EstablishState.java b/router/java/src/net/i2p/router/transport/ntcp/EstablishState.java index dce7d0de7..8de07fe6e 100644 --- a/router/java/src/net/i2p/router/transport/ntcp/EstablishState.java +++ b/router/java/src/net/i2p/router/transport/ntcp/EstablishState.java @@ -234,7 +234,7 @@ public class EstablishState { System.arraycopy(_X, 0, xy, 0, _X.length); System.arraycopy(_Y, 0, xy, _X.length, _Y.length); Hash hxy = _context.sha().calculateHash(xy); - _tsB = _context.clock().now()/1000l; // our (Bob's) timestamp in seconds + _tsB = (_context.clock().now() + 500) / 1000l; // our (Bob's) timestamp in seconds byte padding[] = new byte[12]; // the encrypted data needs an extra 12 bytes _context.random().nextBytes(padding); byte toEncrypt[] = new byte[hxy.getData().length+4+padding.length]; @@ -387,7 +387,7 @@ public class EstablishState { return; } _tsB = DataHelper.fromLong(hXY_tsB, Hash.HASH_LENGTH, 4); // their (Bob's) timestamp in seconds - _tsA = _context.clock().now()/1000; // our (Alice's) timestamp in seconds + _tsA = (_context.clock().now() + 500) / 1000; // our (Alice's) timestamp in seconds if (_log.shouldLog(Log.DEBUG)) _log.debug(prefix()+"h(X+Y) is correct, tsA-tsB=" + (_tsA-_tsB)); diff --git a/router/java/src/net/i2p/router/transport/ntcp/NTCPConnection.java b/router/java/src/net/i2p/router/transport/ntcp/NTCPConnection.java index b161909c8..3fb4c6b37 100644 --- a/router/java/src/net/i2p/router/transport/ntcp/NTCPConnection.java +++ b/router/java/src/net/i2p/router/transport/ntcp/NTCPConnection.java @@ -1035,7 +1035,7 @@ public class NTCPConnection implements FIFOBandwidthLimiter.CompleteListener { * */ private void readMeta(byte unencrypted[]) { - long ourTs = _context.clock().now()/1000; + long ourTs = (_context.clock().now() + 500) / 1000; long ts = DataHelper.fromLong(unencrypted, 2, 4); Adler32 crc = new Adler32(); crc.update(unencrypted, 0, unencrypted.length-4); @@ -1068,7 +1068,7 @@ public class NTCPConnection implements FIFOBandwidthLimiter.CompleteListener { synchronized (_meta) { _context.random().nextBytes(_meta); // randomize the uninterpreted, then overwrite w/ data DataHelper.toLong(_meta, 0, 2, 0); - DataHelper.toLong(_meta, 2, 4, _context.clock().now()/1000); + DataHelper.toLong(_meta, 2, 4, (_context.clock().now() + 500) / 1000); Adler32 crc = new Adler32(); crc.update(_meta, 0, _meta.length-4); DataHelper.toLong(_meta, _meta.length-4, 4, crc.getValue()); From 78a588af0eceba4f203ebe22abeeb5d5ab193312 Mon Sep 17 00:00:00 2001 From: zzz Date: Fri, 26 Nov 2010 14:57:16 +0000 Subject: [PATCH 09/10] reduce UDP clock bias by rounding clock and adjusting for RTT --- .../router/transport/udp/PacketBuilder.java | 2 +- .../i2p/router/transport/udp/PeerState.java | 26 ++++++++++++++++--- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/router/java/src/net/i2p/router/transport/udp/PacketBuilder.java b/router/java/src/net/i2p/router/transport/udp/PacketBuilder.java index 4227aaad0..98daaecf9 100644 --- a/router/java/src/net/i2p/router/transport/udp/PacketBuilder.java +++ b/router/java/src/net/i2p/router/transport/udp/PacketBuilder.java @@ -1084,7 +1084,7 @@ class PacketBuilder { // header data[off] = flagByte; off++; - long now = _context.clock().now() / 1000; + long now = (_context.clock().now() + 500) / 1000; DataHelper.toLong(data, off, 4, now); // todo: add support for rekeying and extended options return packet; diff --git a/router/java/src/net/i2p/router/transport/udp/PeerState.java b/router/java/src/net/i2p/router/transport/udp/PeerState.java index 6c1dec3e4..2c518ed69 100644 --- a/router/java/src/net/i2p/router/transport/udp/PeerState.java +++ b/router/java/src/net/i2p/router/transport/udp/PeerState.java @@ -63,8 +63,13 @@ class PeerState { private boolean _rekeyBeganLocally; /** when were the current cipher and MAC keys established/rekeyed? */ private long _keyEstablishedTime; - /** how far off is the remote peer from our clock, in milliseconds? */ + + /** + * How far off is the remote peer from our clock, in milliseconds? + * A positive number means our clock is ahead of theirs. + */ private long _clockSkew; + /** what is the current receive second, for congestion control? */ private long _currentReceiveSecond; /** when did we last send them a packet? */ @@ -346,8 +351,13 @@ class PeerState { public boolean getRekeyBeganLocally() { return _rekeyBeganLocally; } /** when were the current cipher and MAC keys established/rekeyed? */ public long getKeyEstablishedTime() { return _keyEstablishedTime; } - /** how far off is the remote peer from our clock, in milliseconds? */ + + /** + * How far off is the remote peer from our clock, in milliseconds? + * A positive number means our clock is ahead of theirs. + */ public long getClockSkew() { return _clockSkew ; } + /** what is the current receive second, for congestion control? */ public long getCurrentReceiveSecond() { return _currentReceiveSecond; } /** when did we last send them a packet? */ @@ -444,10 +454,17 @@ class PeerState { public void setRekeyBeganLocally(boolean local) { _rekeyBeganLocally = local; } /** when were the current cipher and MAC keys established/rekeyed? */ public void setKeyEstablishedTime(long when) { _keyEstablishedTime = when; } - /** how far off is the remote peer from our clock, in milliseconds? */ + + /** + * Update the moving-average clock skew based on the current difference. + * The raw skew will be adjusted for RTT/2 here. + * @param skew milliseconds, NOT adjusted for RTT. + * A positive number means our clock is ahead of theirs. + */ public void adjustClockSkew(long skew) { - _clockSkew = (long) (0.9*(float)_clockSkew + 0.1*(float)skew); + _clockSkew = (long) (0.9*(float)_clockSkew + 0.1*(float)(skew - (_rtt / 2))); } + /** what is the current receive second, for congestion control? */ public void setCurrentReceiveSecond(long sec) { _currentReceiveSecond = sec; } /** when did we last send them a packet? */ @@ -679,6 +696,7 @@ class PeerState { * */ public List getCurrentFullACKs() { + // no such element exception seen here ArrayList rv = new ArrayList(_currentACKs); // include some for retransmission rv.addAll(_currentACKsResend); From a0a3622f16999645a6a51cf35c4debee094162e5 Mon Sep 17 00:00:00 2001 From: zzz Date: Fri, 26 Nov 2010 15:04:58 +0000 Subject: [PATCH 10/10] remove unneeded initializers --- .../transport/udp/InboundEstablishState.java | 3 -- .../transport/udp/OutboundEstablishState.java | 2 -- .../udp/OutboundMessageFragments.java | 1 - .../transport/udp/OutboundMessageState.java | 3 -- .../i2p/router/transport/udp/PeerState.java | 28 ------------------- .../router/transport/udp/PeerTestManager.java | 2 -- .../TimedWeightedPriorityMessageQueue.java | 1 - .../router/transport/udp/UDPTransport.java | 1 - 8 files changed, 41 deletions(-) diff --git a/router/java/src/net/i2p/router/transport/udp/InboundEstablishState.java b/router/java/src/net/i2p/router/transport/udp/InboundEstablishState.java index ea40c616c..334a075fb 100644 --- a/router/java/src/net/i2p/router/transport/udp/InboundEstablishState.java +++ b/router/java/src/net/i2p/router/transport/udp/InboundEstablishState.java @@ -72,9 +72,6 @@ class InboundEstablishState { _alicePort = remotePort; _remoteHostId = new RemoteHostId(_aliceIP, _alicePort); _bobPort = localPort; - _keyBuilder = null; - _verificationAttempted = false; - _complete = false; _currentState = STATE_UNKNOWN; _establishBegin = ctx.clock().now(); } diff --git a/router/java/src/net/i2p/router/transport/udp/OutboundEstablishState.java b/router/java/src/net/i2p/router/transport/udp/OutboundEstablishState.java index 1e23a210c..76753a5f5 100644 --- a/router/java/src/net/i2p/router/transport/udp/OutboundEstablishState.java +++ b/router/java/src/net/i2p/router/transport/udp/OutboundEstablishState.java @@ -86,13 +86,11 @@ class OutboundEstablishState { } _remotePeer = remotePeer; _introKey = introKey; - _keyBuilder = null; _queuedMessages = new LinkedBlockingQueue(); _currentState = STATE_UNKNOWN; _establishBegin = ctx.clock().now(); _remoteAddress = addr; _introductionNonce = -1; - _complete = false; prepareSessionRequest(); if ( (addr != null) && (addr.getIntroducerCount() > 0) ) { if (_log.shouldLog(Log.DEBUG)) diff --git a/router/java/src/net/i2p/router/transport/udp/OutboundMessageFragments.java b/router/java/src/net/i2p/router/transport/udp/OutboundMessageFragments.java index e25ab0369..58e700f09 100644 --- a/router/java/src/net/i2p/router/transport/udp/OutboundMessageFragments.java +++ b/router/java/src/net/i2p/router/transport/udp/OutboundMessageFragments.java @@ -47,7 +47,6 @@ class OutboundMessageFragments { _transport = transport; // _throttle = throttle; _activePeers = new ArrayList(256); - _nextPeer = 0; _builder = new PacketBuilder(ctx, transport); _alive = true; // _allowExcess = false; diff --git a/router/java/src/net/i2p/router/transport/udp/OutboundMessageState.java b/router/java/src/net/i2p/router/transport/udp/OutboundMessageState.java index a545cfb06..0e5ed7ef3 100644 --- a/router/java/src/net/i2p/router/transport/udp/OutboundMessageState.java +++ b/router/java/src/net/i2p/router/transport/udp/OutboundMessageState.java @@ -47,9 +47,6 @@ class OutboundMessageState { public OutboundMessageState(I2PAppContext context) { _context = context; _log = _context.logManager().getLog(OutboundMessageState.class); - _pushCount = 0; - _maxSends = 0; - // _nextSendFragment = 0; } public boolean initialize(OutNetMessage msg) { diff --git a/router/java/src/net/i2p/router/transport/udp/PeerState.java b/router/java/src/net/i2p/router/transport/udp/PeerState.java index 2c518ed69..54ece24c2 100644 --- a/router/java/src/net/i2p/router/transport/udp/PeerState.java +++ b/router/java/src/net/i2p/router/transport/udp/PeerState.java @@ -246,36 +246,19 @@ class PeerState { _context = ctx; _log = ctx.logManager().getLog(PeerState.class); _transport = transport; - _remotePeer = null; - _currentMACKey = null; - _currentCipherKey = null; - _nextMACKey = null; - _nextCipherKey = null; - _nextKeyingMaterial = null; - _rekeyBeganLocally = false; _keyEstablishedTime = -1; - _clockSkew = 0; _currentReceiveSecond = -1; _lastSendTime = -1; _lastReceiveTime = -1; _currentACKs = new ConcurrentHashSet(); _currentACKsResend = new LinkedBlockingQueue(); - _currentSecondECNReceived = false; - _remoteWantsPreviousACKs = false; _sendWindowBytes = DEFAULT_SEND_WINDOW_BYTES; _sendWindowBytesRemaining = DEFAULT_SEND_WINDOW_BYTES; _slowStartThreshold = MAX_SEND_WINDOW_BYTES/2; _lastSendRefill = _context.clock().now(); _receivePeriodBegin = _lastSendRefill; - _sendBps = 0; - _sendBytes = 0; - _receiveBps = 0; _lastCongestionOccurred = -1; - _remoteIP = null; _remotePort = -1; - _remoteRequiresIntroduction = false; - _weRelayToThemAs = 0; - _theyRelayToUsAs = 0; _mtu = getDefaultMTU(); _mtuReceive = _mtu; _mtuLastChecked = -1; @@ -283,19 +266,8 @@ class PeerState { _rto = MIN_RTO; _rtt = _rto/2; _rttDeviation = _rtt; - _messagesReceived = 0; - _messagesSent = 0; - _packetsTransmitted = 0; - _packetsRetransmitted = 0; - _packetRetransmissionRate = 0; - _retransmissionPeriodStart = 0; - _packetsReceived = 0; - _packetsReceivedDuplicate = 0; _inboundMessages = new HashMap(8); _outboundMessages = new ArrayList(32); - _dead = false; - _isInbound = false; - _lastIntroducerTime = 0; _context.statManager().createRateStat("udp.congestionOccurred", "How large the cwin was when congestion occurred (duration == sendBps)", "udp", UDPTransport.RATES); _context.statManager().createRateStat("udp.congestedRTO", "retransmission timeout after congestion (duration == rtt dev)", "udp", UDPTransport.RATES); _context.statManager().createRateStat("udp.sendACKPartial", "Number of partial ACKs sent (duration == number of full ACKs in that ack packet)", "udp", UDPTransport.RATES); diff --git a/router/java/src/net/i2p/router/transport/udp/PeerTestManager.java b/router/java/src/net/i2p/router/transport/udp/PeerTestManager.java index aa145bbe9..a3211a5a6 100644 --- a/router/java/src/net/i2p/router/transport/udp/PeerTestManager.java +++ b/router/java/src/net/i2p/router/transport/udp/PeerTestManager.java @@ -118,8 +118,6 @@ class PeerTestManager { _activeTests = new ConcurrentHashMap(); _recentTests = new LinkedBlockingQueue(); _packetBuilder = new PacketBuilder(context, transport); - _currentTest = null; - _currentTestComplete = false; _context.statManager().createRateStat("udp.statusKnownCharlie", "How often the bob we pick passes us to a charlie we already have a session with?", "udp", UDPTransport.RATES); _context.statManager().createRateStat("udp.receiveTestReply", "How often we get a reply to our peer test?", "udp", UDPTransport.RATES); _context.statManager().createRateStat("udp.receiveTest", "How often we get a packet requesting us to participate in a peer test?", "udp", UDPTransport.RATES); diff --git a/router/java/src/net/i2p/router/transport/udp/TimedWeightedPriorityMessageQueue.java b/router/java/src/net/i2p/router/transport/udp/TimedWeightedPriorityMessageQueue.java index 9721f22cc..86fe2c811 100644 --- a/router/java/src/net/i2p/router/transport/udp/TimedWeightedPriorityMessageQueue.java +++ b/router/java/src/net/i2p/router/transport/udp/TimedWeightedPriorityMessageQueue.java @@ -76,7 +76,6 @@ class TimedWeightedPriorityMessageQueue implements MessageQueue, OutboundMessage } _alive = true; _nextLock = this; - _nextQueue = 0; _chokedPeers = Collections.synchronizedSet(new HashSet(16)); _listener = lsnr; _context.statManager().createRateStat("udp.timeToEntrance", "Message lifetime until it reaches the UDP system", "udp", UDPTransport.RATES); diff --git a/router/java/src/net/i2p/router/transport/udp/UDPTransport.java b/router/java/src/net/i2p/router/transport/udp/UDPTransport.java index dc0790ac4..e287d1426 100644 --- a/router/java/src/net/i2p/router/transport/udp/UDPTransport.java +++ b/router/java/src/net/i2p/router/transport/udp/UDPTransport.java @@ -177,7 +177,6 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority _peersByIdent = new ConcurrentHashMap(128); _peersByRemoteHost = new ConcurrentHashMap(128); _dropList = new ConcurrentHashSet(2); - _endpoint = null; // See comments in DQAT.java if (USE_PRIORITY) {