From 1aba32448161049480263853493d9acb0cf52e65 Mon Sep 17 00:00:00 2001 From: zzz Date: Fri, 15 Oct 2010 13:41:56 +0000 Subject: [PATCH 01/31] * I2PSocketEepGet: Set connect delay to save a RTT, will speed announces in i2psnark --- .../src/net/i2p/client/streaming/I2PSocketEepGet.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/apps/ministreaming/java/src/net/i2p/client/streaming/I2PSocketEepGet.java b/apps/ministreaming/java/src/net/i2p/client/streaming/I2PSocketEepGet.java index 8aac29cff..834c52160 100644 --- a/apps/ministreaming/java/src/net/i2p/client/streaming/I2PSocketEepGet.java +++ b/apps/ministreaming/java/src/net/i2p/client/streaming/I2PSocketEepGet.java @@ -45,6 +45,10 @@ public class I2PSocketEepGet extends EepGet { /** this replaces _proxy in the superclass. Sadly, I2PSocket does not extend Socket. */ private I2PSocket _socket; + /** from ConnectionOptions */ + private static final String PROP_CONNECT_DELAY = "i2p.streaming.connectDelay"; + private static final String CONNECT_DELAY = "500"; + public I2PSocketEepGet(I2PAppContext ctx, I2PSocketManager mgr, int numRetries, String outputFile, String url) { this(ctx, mgr, numRetries, -1, -1, outputFile, null, url); } @@ -123,6 +127,10 @@ public class I2PSocketEepGet extends EepGet { Properties props = new Properties(); props.setProperty(I2PSocketOptions.PROP_CONNECT_TIMEOUT, "" + CONNECT_TIMEOUT); props.setProperty(I2PSocketOptions.PROP_READ_TIMEOUT, "" + INACTIVITY_TIMEOUT); + // This is important - even if the underlying socket doesn't have a connect delay, + // we want to set it for this connection, so the request headers will go out + // in the SYN packet, saving one RTT. + props.setProperty(PROP_CONNECT_DELAY, CONNECT_DELAY); I2PSocketOptions opts = _socketManager.buildOptions(props); _socket = _socketManager.connect(dest, opts); } else { From 9afff4f80acb1421d41e1eaa59f9d997ff752ca6 Mon Sep 17 00:00:00 2001 From: zzz Date: Fri, 15 Oct 2010 13:48:36 +0000 Subject: [PATCH 02/31] * i2psnark: Add file priority feature; Use context random for shuffle; other cleanups --- .../src/org/klomp/snark/I2PSnarkUtil.java | 4 + .../src/org/klomp/snark/PeerCoordinator.java | 84 ++++++++++++++-- .../java/src/org/klomp/snark/Piece.java | 32 ++++++- .../java/src/org/klomp/snark/Storage.java | 96 ++++++++++++++++++- .../src/org/klomp/snark/TrackerClient.java | 2 +- .../org/klomp/snark/web/I2PSnarkServlet.java | 78 +++++++++++++-- 6 files changed, 271 insertions(+), 25 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java index 6f17a810a..195659c50 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java +++ b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java @@ -163,6 +163,10 @@ public class I2PSnarkUtil { opts.setProperty("inbound.nickname", "I2PSnark"); if (opts.getProperty("outbound.nickname") == null) opts.setProperty("outbound.nickname", "I2PSnark"); + // Dont do this for now, it is set in I2PSocketEepGet for announces, + // we don't need fast handshake for peer connections. + //if (opts.getProperty("i2p.streaming.connectDelay") == null) + // opts.setProperty("i2p.streaming.connectDelay", "500"); if (opts.getProperty("i2p.streaming.inactivityTimeout") == null) opts.setProperty("i2p.streaming.inactivityTimeout", "240000"); if (opts.getProperty("i2p.streaming.inactivityAction") == null) diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java index 8f24c864a..07c832440 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java @@ -78,6 +78,7 @@ public class PeerCoordinator implements PeerListener private final CoordinatorListener listener; public I2PSnarkUtil _util; + private static final Random _random = I2PAppContext.getGlobalContext().random(); public String trackerProblems = null; public int trackerSeenPeers = 0; @@ -97,8 +98,7 @@ public class PeerCoordinator implements PeerListener // Install a timer to check the uploaders. // Randomize the first start time so multiple tasks are spread out, // this will help the behavior with global limits - Random r = I2PAppContext.getGlobalContext().random(); - timer.schedule(new PeerCheckerTask(_util, this), (CHECK_PERIOD / 2) + r.nextInt((int) CHECK_PERIOD), CHECK_PERIOD); + timer.schedule(new PeerCheckerTask(_util, this), (CHECK_PERIOD / 2) + _random.nextInt((int) CHECK_PERIOD), CHECK_PERIOD); } // only called externally from Storage after the double-check fails @@ -107,10 +107,16 @@ public class PeerCoordinator implements PeerListener // Make a list of pieces wantedPieces = new ArrayList(); BitField bitfield = storage.getBitField(); - for(int i = 0; i < metainfo.getPieces(); i++) - if (!bitfield.get(i)) - wantedPieces.add(new Piece(i)); - Collections.shuffle(wantedPieces); + int[] pri = storage.getPiecePriorities(); + for(int i = 0; i < metainfo.getPieces(); i++) { + if (!bitfield.get(i)) { + Piece p = new Piece(i); + if (pri != null) + p.setPriority(pri[i]); + wantedPieces.add(p); + } + } + Collections.shuffle(wantedPieces, _random); } public Storage getStorage() { return storage; } @@ -520,6 +526,9 @@ public class PeerCoordinator implements PeerListener while (piece == null && it.hasNext()) { Piece p = it.next(); + // sorted by priority, so when we hit a disabled piece we are done + if (p.isDisabled()) + break; if (havePieces.get(p.getId()) && !p.isRequested()) { piece = p; @@ -538,7 +547,7 @@ public class PeerCoordinator implements PeerListener if (wantedPieces.size() > END_GAME_THRESHOLD) return -1; // nothing to request and not in end game // let's not all get on the same piece - Collections.shuffle(requested); + Collections.shuffle(requested, _random); Iterator it2 = requested.iterator(); while (piece == null && it2.hasNext()) { @@ -563,11 +572,64 @@ public class PeerCoordinator implements PeerListener _log.debug("parallel request (end game?) for " + peer + ": piece = " + piece); } } + if (_log.shouldLog(Log.DEBUG)) + _log.debug("Now requesting: piece " + piece + " priority " + piece.getPriority()); piece.setRequested(true); return piece.getId(); } } + /** + * Maps file priorities to piece priorities. + * Call after updating file priorities Storage.setPriority() + * @since 0.8.1 + */ + public void updatePiecePriorities() { + int[] pri = storage.getPiecePriorities(); + if (pri == null) + return; + synchronized(wantedPieces) { + // Add incomplete and previously unwanted pieces to the list + // Temp to avoid O(n**2) + BitField want = new BitField(pri.length); + for (Piece p : wantedPieces) { + want.set(p.getId()); + } + BitField bitfield = storage.getBitField(); + for (int i = 0; i < pri.length; i++) { + if (pri[i] >= 0 && !bitfield.get(i)) { + if (!want.get(i)) { + Piece piece = new Piece(i); + wantedPieces.add(piece); + // As connections are already up, new Pieces will + // not have their PeerID list populated, so do that. + synchronized(peers) { + for (Peer p : peers) { + PeerState s = p.state; + if (s != null) { + BitField bf = s.bitfield; + if (bf != null && bf.get(i)) + piece.addPeer(p); + } + } + } + } + } + } + // now set the new priorities and remove newly unwanted pieces + for (Iterator iter = wantedPieces.iterator(); iter.hasNext(); ) { + Piece p = iter.next(); + int id = pri[p.getId()]; + if (id >= 0) + p.setPriority(pri[p.getId()]); + else + iter.remove(); + } + // if we added pieces, they will be in-order unless we shuffle + Collections.shuffle(wantedPieces, _random); + } + } + /** * Returns a byte array containing the requested piece or null of * the piece is unknown. @@ -632,14 +694,18 @@ public class PeerCoordinator implements PeerListener // No need to announce have piece to peers. // Assume we got a good piece, we don't really care anymore. - return true; + // Well, this could be caused by a change in priorities, so + // only return true if we already have it, otherwise might as well keep it. + if (storage.getBitField().get(piece)) + return true; } try { if (storage.putPiece(piece, bs)) { - _log.info("Got valid piece " + piece + "/" + metainfo.getPieces() +" from " + peer + " for " + metainfo.getName()); + if (_log.shouldLog(Log.INFO)) + _log.info("Got valid piece " + piece + "/" + metainfo.getPieces() +" from " + peer + " for " + metainfo.getName()); } else { diff --git a/apps/i2psnark/java/src/org/klomp/snark/Piece.java b/apps/i2psnark/java/src/org/klomp/snark/Piece.java index 3fd0771f9..0ae9570e1 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Piece.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Piece.java @@ -1,22 +1,31 @@ package org.klomp.snark; import java.util.Collections; -import java.util.HashSet; import java.util.Set; +import net.i2p.util.ConcurrentHashSet; + public class Piece implements Comparable { private int id; - private Set peers; + private Set peers; private boolean requested; + /** @since 0.8.1 */ + private int priority; public Piece(int id) { this.id = id; - this.peers = Collections.synchronizedSet(new HashSet()); - this.requested = false; + this.peers = new ConcurrentHashSet(); } + /** + * Highest priority first, + * then rarest first + */ public int compareTo(Object o) throws ClassCastException { + int pdiff = ((Piece)o).priority - this.priority; // reverse + if (pdiff != 0) + return pdiff; return this.peers.size() - ((Piece)o).peers.size(); } @@ -37,12 +46,25 @@ public class Piece implements Comparable { } public int getId() { return this.id; } - public Set getPeers() { return this.peers; } + /** @deprecated unused */ + public Set getPeers() { return this.peers; } public boolean addPeer(Peer peer) { return this.peers.add(peer.getPeerID()); } public boolean removePeer(Peer peer) { return this.peers.remove(peer.getPeerID()); } public boolean isRequested() { return this.requested; } public void setRequested(boolean requested) { this.requested = requested; } + /** @return default 0 @since 0.8.1 */ + public int getPriority() { return this.priority; } + + /** @since 0.8.1 */ + public void setPriority(int p) { this.priority = p; } + + /** @since 0.8.1 */ + public boolean isDisabled() { return this.priority < 0; } + + /** @since 0.8.1 */ + public void setDisabled() { this.priority = -1; } + @Override public String toString() { return String.valueOf(id); diff --git a/apps/i2psnark/java/src/org/klomp/snark/Storage.java b/apps/i2psnark/java/src/org/klomp/snark/Storage.java index f90301848..1e8fb5ec2 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Storage.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Storage.java @@ -42,6 +42,8 @@ public class Storage private Object[] RAFlock; // lock on RAF access private long[] RAFtime; // when was RAF last accessed, or 0 if closed private File[] RAFfile; // File to make it easier to reopen + /** priorities by file; default 0; may be null. @since 0.8.1 */ + private int[] priorities; private final StorageListener listener; private I2PSnarkUtil _util; @@ -228,6 +230,8 @@ public class Storage RAFlock = new Object[size]; RAFtime = new long[size]; RAFfile = new File[size]; + priorities = new int[size]; + int i = 0; Iterator it = files.iterator(); @@ -330,6 +334,83 @@ public class Storage return -1; } + /** + * @param file canonical path (non-directory) + * @since 0.8.1 + */ + public int getPriority(String file) { + if (complete() || metainfo.getFiles() == null || priorities == null) + return 0; + for (int i = 0; i < rafs.length; i++) { + File f = RAFfile[i]; + // use canonical in case snark dir or sub dirs are symlinked + if (f != null) { + try { + String canonical = f.getCanonicalPath(); + if (canonical.equals(file)) + return priorities[i]; + } catch (IOException ioe) {} + } + } + return 0; + } + + /** + * Must call setPiecePriorities() after calling this + * @param file canonical path (non-directory) + * @param priority default 0; <0 to disable + * @since 0.8.1 + */ + public void setPriority(String file, int pri) { + if (complete() || metainfo.getFiles() == null || priorities == null) + return; + for (int i = 0; i < rafs.length; i++) { + File f = RAFfile[i]; + // use canonical in case snark dir or sub dirs are symlinked + if (f != null) { + try { + String canonical = f.getCanonicalPath(); + if (canonical.equals(file)) { + priorities[i] = pri; + return; + } + } catch (IOException ioe) {} + } + } + } + + /** + * Call setPriority() for all changed files first, + * then call this. + * Set the piece priority to the highest priority + * of all files spanning the piece. + * Caller must pass array to the PeerCoordinator. + * @return null on error, if complete, or if only one file + * @since 0.8.1 + */ + public int[] getPiecePriorities() { + if (complete() || metainfo.getFiles() == null) + return null; + int[] rv = new int[metainfo.getPieces()]; + int file = 0; + long pcEnd = -1; + long fileEnd = lengths[0] - 1; + int psz = metainfo.getPieceLength(0); + for (int i = 0; i < rv.length; i++) { + pcEnd += psz; + int pri = priorities[file]; + while (fileEnd <= pcEnd && file < lengths.length - 1) { + file++; + long oldFileEnd = fileEnd; + fileEnd += lengths[file]; + if (priorities[file] > pri && pcEnd < oldFileEnd) + pri = priorities[file]; + } + rv[i] = pri; + } + return rv; + } + /** * The BitField that tells which pieces this storage contains. * Do not change this since this is the current state of the storage. @@ -436,10 +517,14 @@ public class Storage changed = true; checkCreateFiles(); } - if (complete()) + if (complete()) { _util.debug("Torrent is complete", Snark.NOTICE); - else + } else { + // fixme saved priorities + if (files != null) + priorities = new int[files.size()]; _util.debug("Still need " + needed + " out of " + metainfo.getPieces() + " pieces", Snark.NOTICE); + } } /** @@ -624,7 +709,12 @@ public class Storage // the whole file? listener.storageCreateFile(this, names[nr], lengths[nr]); final int ZEROBLOCKSIZE = metainfo.getPieceLength(0); - byte[] zeros = new byte[ZEROBLOCKSIZE]; + byte[] zeros; + try { + zeros = new byte[ZEROBLOCKSIZE]; + } catch (OutOfMemoryError oom) { + throw new IOException(oom.toString()); + } int i; for (i = 0; i < lengths[nr]/ZEROBLOCKSIZE; i++) { diff --git a/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java b/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java index 05df351c6..d904f98ca 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java +++ b/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java @@ -266,7 +266,7 @@ public class TrackerClient extends I2PAppThread // we only want to talk to new people if we need things // from them (duh) List ordered = new ArrayList(peers); - Collections.shuffle(ordered); + Collections.shuffle(ordered, r); Iterator it = ordered.iterator(); while (it.hasNext()) { Peer cur = (Peer)it.next(); diff --git a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java index d3a04df1f..1c9767800 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java +++ b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java @@ -133,6 +133,7 @@ public class I2PSnarkServlet extends Default { // bypass the horrid Resource.getListHTML() String pathInfo = req.getPathInfo(); String pathInContext = URI.addPaths(path, pathInfo); + req.setCharacterEncoding("UTF-8"); resp.setCharacterEncoding("UTF-8"); resp.setContentType("text/html; charset=UTF-8"); Resource resource = getResource(pathInContext); @@ -140,7 +141,7 @@ public class I2PSnarkServlet extends Default { resp.sendError(HttpResponse.__404_Not_Found); } else { String base = URI.addPaths(req.getRequestURI(), "/"); - String listing = getListHTML(resource, base, true); + String listing = getListHTML(resource, base, true, method.equals("POST") ? req.getParameterMap() : null); if (listing != null) resp.getWriter().write(listing); else // shouldn't happen @@ -1252,10 +1253,11 @@ public class I2PSnarkServlet extends Default { * @param r The Resource * @param base The base URL * @param parent True if the parent directory should be included + * @param postParams map of POST parameters or null if not a POST * @return String of HTML * @since 0.7.14 */ - private String getListHTML(Resource r, String base, boolean parent) + private String getListHTML(Resource r, String base, boolean parent, Map postParams) throws IOException { if (!r.isDirectory()) @@ -1280,6 +1282,10 @@ public class I2PSnarkServlet extends Default { else torrentName = title; Snark snark = _manager.getTorrentByBaseName(torrentName); + + if (snark != null && postParams != null) + savePriorities(snark, postParams); + if (title.endsWith("/")) title = title.substring(0, title.length() - 1); title = _("Torrent") + ": " + title; @@ -1297,12 +1303,19 @@ public class I2PSnarkServlet extends Default { .append(_("Up to higher level directory")).append("\n"); } - buf.append("
" + - "" + + buf.append("
"); + boolean showPriority = snark != null && !snark.storage.complete(); + if (showPriority) + buf.append("
\n"); + buf.append("
" + ""); + .append(""); + if (showPriority) + buf.append(""); + buf.append("\n"); //DateFormat dfmt=DateFormat.getDateTimeInstance(DateFormat.MEDIUM, // DateFormat.MEDIUM); + boolean showSaveButton = false; for (int i=0 ; i< ls.length ; i++) { String encoded=URI.encodePath(ls[i]); @@ -1340,7 +1353,8 @@ public class I2PSnarkServlet extends Default { complete = true; status = toImg("tick") + _("Complete"); } else { - status = toImg("clock") + + status = + (snark.storage.getPriority(f.getCanonicalPath()) < 0 ? toImg("cancel") : toImg("clock")) + (100 * (length - remaining) / length) + "% " + _("complete") + " (" + DataHelper.formatSize2(remaining) + _("bytes remaining") + ")"; } @@ -1384,9 +1398,40 @@ public class I2PSnarkServlet extends Default { buf.append("\n"); + buf.append(""); + if (showPriority) { + buf.append(""); + } + buf.append("\n"); + } + if (showSaveButton) { + buf.append("\n"); } buf.append("
").append(_("File")).append("").append(_("Size")) - .append("").append(_("Status")).append("
").append(_("Status")).append("").append(_("Priority")).append("
"); //buf.append(dfmt.format(new Date(item.lastModified()))); buf.append(status); - buf.append("
"); + File f = item.getFile(); + if ((!complete) && (!item.isDirectory()) && f != null) { + int pri = snark.storage.getPriority(f.getCanonicalPath()); + buf.append(" 0) + buf.append("checked=\"true\""); + buf.append('>').append(_("High")); + + buf.append("').append(_("Normal")); + + buf.append("').append(_("Do not download")); + showSaveButton = true; + } + buf.append("
 
\n"); + if (showPriority) + buf.append(""); buf.append("
\n"); return buf.toString(); @@ -1452,6 +1497,25 @@ public class I2PSnarkServlet extends Default { return "\"\" "; } + /** @since 0.8.1 */ + private static void savePriorities(Snark snark, Map postParams) { + Set entries = postParams.entrySet(); + for (Map.Entry entry : entries) { + String key = (String)entry.getKey(); + if (key.startsWith("pri.")) { + try { + String file = key.substring(4); + String val = ((String[])entry.getValue())[0]; // jetty arrays + int pri = Integer.parseInt(val); + snark.storage.setPriority(file, pri); + //System.err.println("Priority now " + pri + " for " + file); + } catch (Throwable t) { t.printStackTrace(); } + } + } + if (snark.coordinator != null) + snark.coordinator.updatePiecePriorities(); + } + /** inner class, don't bother reindenting */ private static class FetchAndAdd implements Runnable { From 24dd78394b83d1eba9684fb2ea4e202668a45541 Mon Sep 17 00:00:00 2001 From: zzz Date: Fri, 15 Oct 2010 17:25:45 +0000 Subject: [PATCH 03/31] priority persistence --- .../src/org/klomp/snark/SnarkManager.java | 63 ++++++++++++++++++- .../java/src/org/klomp/snark/Storage.java | 21 ++++++- .../org/klomp/snark/web/I2PSnarkServlet.java | 5 +- 3 files changed, 84 insertions(+), 5 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java index b0fe6d1ba..240515e59 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java +++ b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java @@ -54,6 +54,7 @@ public class SnarkManager implements Snark.CompleteListener { public static final String PROP_DIR = "i2psnark.dir"; public static final String PROP_META_PREFIX = "i2psnark.zmeta."; public static final String PROP_META_BITFIELD_SUFFIX = ".bitfield"; + public static final String PROP_META_PRIORITY_SUFFIX = ".priority"; private static final String CONFIG_FILE = "i2psnark.config"; public static final String PROP_AUTO_START = "i2snark.autoStart"; // oops @@ -510,6 +511,7 @@ public class SnarkManager implements Snark.CompleteListener { torrent = new Snark(_util, filename, null, -1, null, null, this, _peerCoordinatorSet, _connectionAcceptor, false, dataDir.getPath()); + loadSavedFilePriorities(torrent); torrent.completeListener = this; synchronized (_snarks) { _snarks.put(filename, torrent); @@ -587,6 +589,33 @@ public class SnarkManager implements Snark.CompleteListener { return new BitField(bitfield, len); } + /** + * Get the saved priorities for a torrent from the config file. + * @since 0.8.1 + */ + public void loadSavedFilePriorities(Snark snark) { + MetaInfo metainfo = snark.meta; + if (metainfo.getFiles() == null) + return; + byte[] ih = metainfo.getInfoHash(); + String infohash = Base64.encode(ih); + infohash = infohash.replace('=', '$'); + String pri = _config.getProperty(PROP_META_PREFIX + infohash + PROP_META_PRIORITY_SUFFIX); + if (pri == null) + return; + int filecount = metainfo.getFiles().size(); + int[] rv = new int[filecount]; + String[] arr = pri.split(","); + for (int i = 0; i < filecount && i < arr.length; i++) { + if (arr[i].length() > 0) { + try { + rv[i] = Integer.parseInt(arr[i]); + } catch (Throwable t) {} + } + } + snark.storage.setFilePriorities(rv); + } + /** * Save the completion status of a torrent and the current time in the config file * in the form "i2psnark.zmeta.$base64infohash=$time,$base64bitfield". @@ -595,8 +624,9 @@ public class SnarkManager implements Snark.CompleteListener { * The time is a standard long converted to string. * The status is either a bitfield converted to Base64 or "." for a completed * torrent to save space in the config file and in memory. + * @param priorities may be null */ - public void saveTorrentStatus(MetaInfo metainfo, BitField bitfield) { + public void saveTorrentStatus(MetaInfo metainfo, BitField bitfield, int[] priorities) { byte[] ih = metainfo.getInfoHash(); String infohash = Base64.encode(ih); infohash = infohash.replace('=', '$'); @@ -609,6 +639,34 @@ public class SnarkManager implements Snark.CompleteListener { bfs = Base64.encode(bf); } _config.setProperty(PROP_META_PREFIX + infohash + PROP_META_BITFIELD_SUFFIX, now + "," + bfs); + + // now the file priorities + String prop = PROP_META_PREFIX + infohash + PROP_META_PRIORITY_SUFFIX; + if (priorities != null) { + boolean nonzero = false; + for (int i = 0; i < priorities.length; i++) { + if (priorities[i] != 0) { + nonzero = true; + break; + } + } + if (nonzero) { + // generate string like -5,,4,3,,,,,,-2 where no number is zero. + StringBuilder buf = new StringBuilder(2 * priorities.length); + for (int i = 0; i < priorities.length; i++) { + if (priorities[i] != 0) + buf.append(Integer.toString(priorities[i])); + if (i != priorities.length - 1) + buf.append(','); + } + _config.setProperty(prop, buf.toString()); + } else { + _config.remove(prop); + } + } else { + _config.remove(prop); + } + saveConfig(); } @@ -621,6 +679,7 @@ public class SnarkManager implements Snark.CompleteListener { String infohash = Base64.encode(ih); infohash = infohash.replace('=', '$'); _config.remove(PROP_META_PREFIX + infohash + PROP_META_BITFIELD_SUFFIX); + _config.remove(PROP_META_PREFIX + infohash + PROP_META_PRIORITY_SUFFIX); saveConfig(); } @@ -742,7 +801,7 @@ public class SnarkManager implements Snark.CompleteListener { } public void updateStatus(Snark snark) { - saveTorrentStatus(snark.meta, snark.storage.getBitField()); + saveTorrentStatus(snark.meta, snark.storage.getBitField(), snark.storage.getFilePriorities()); } private void monitorTorrents(File dir) { diff --git a/apps/i2psnark/java/src/org/klomp/snark/Storage.java b/apps/i2psnark/java/src/org/klomp/snark/Storage.java index 1e8fb5ec2..d208adc34 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Storage.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Storage.java @@ -379,6 +379,25 @@ public class Storage } } + /** + * Get the file priorities array. + * @return null on error, if complete, or if only one file + * @since 0.8.1 + */ + public int[] getFilePriorities() { + return priorities; + } + + /** + * Set the file priorities array. + * Only call this when stopped, but after check() + * @param p may be null + * @since 0.8.1 + */ + void setFilePriorities(int[] p) { + priorities = p; + } + /** * Call setPriority() for all changed files first, * then call this. @@ -389,7 +408,7 @@ public class Storage * @since 0.8.1 */ public int[] getPiecePriorities() { - if (complete() || metainfo.getFiles() == null) + if (complete() || metainfo.getFiles() == null || priorities == null) return null; int[] rv = new int[metainfo.getPieces()]; int file = 0; diff --git a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java index 1c9767800..19fb3bd0f 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java +++ b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java @@ -503,7 +503,7 @@ public class I2PSnarkServlet extends Default { File torrentFile = new File(baseFile.getParent(), baseFile.getName() + ".torrent"); if (torrentFile.exists()) throw new IOException("Cannot overwrite an existing .torrent file: " + torrentFile.getPath()); - _manager.saveTorrentStatus(info, s.getBitField()); // so addTorrent won't recheck + _manager.saveTorrentStatus(info, s.getBitField(), null); // so addTorrent won't recheck // DirMonitor could grab this first, maybe hold _snarks lock? FileOutputStream out = new FileOutputStream(torrentFile); out.write(info.getTorrentData()); @@ -1498,7 +1498,7 @@ public class I2PSnarkServlet extends Default { } /** @since 0.8.1 */ - private static void savePriorities(Snark snark, Map postParams) { + private void savePriorities(Snark snark, Map postParams) { Set entries = postParams.entrySet(); for (Map.Entry entry : entries) { String key = (String)entry.getKey(); @@ -1514,6 +1514,7 @@ public class I2PSnarkServlet extends Default { } if (snark.coordinator != null) snark.coordinator.updatePiecePriorities(); + _manager.saveTorrentStatus(snark.storage.getMetaInfo(), snark.storage.getBitField(), snark.storage.getFilePriorities()); } From 1400c4d4d0992a262670cd9fbd2b076da3f5ea65 Mon Sep 17 00:00:00 2001 From: echelon Date: Mon, 18 Oct 2010 08:28:15 +0000 Subject: [PATCH 04/31] added plurals to fr, de and nl po file --- apps/routerconsole/locale/messages_de.po | 1 + apps/routerconsole/locale/messages_fr.po | 4 ++-- apps/routerconsole/locale/messages_nl.po | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/routerconsole/locale/messages_de.po b/apps/routerconsole/locale/messages_de.po index 40ebdebd6..f58bb8e0e 100644 --- a/apps/routerconsole/locale/messages_de.po +++ b/apps/routerconsole/locale/messages_de.po @@ -15,6 +15,7 @@ msgstr "" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1)\n" "X-Poedit-Language: German\n" #: ../../../router/java/src/net/i2p/router/Blocklist.java:126 diff --git a/apps/routerconsole/locale/messages_fr.po b/apps/routerconsole/locale/messages_fr.po index 123a9adb8..e550c8b65 100644 --- a/apps/routerconsole/locale/messages_fr.po +++ b/apps/routerconsole/locale/messages_fr.po @@ -15,8 +15,8 @@ msgstr "" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"X-Poedit-Language: German\n" -"X-Poedit-Basepath: /home/lee/work/i2p/monotone/i2p.i2p/apps/routerconsole/java\n" +"X-Poedit-Language: French\n" +"Plural-Forms: nplurals=2; plural=(n != 1)\n" #: ../jsp/WEB-INF/classes/net/i2p/router/web/jsp/config_jsp.java:106 msgid "config networking" diff --git a/apps/routerconsole/locale/messages_nl.po b/apps/routerconsole/locale/messages_nl.po index ffaa50d17..52aaf08d8 100644 --- a/apps/routerconsole/locale/messages_nl.po +++ b/apps/routerconsole/locale/messages_nl.po @@ -15,6 +15,7 @@ msgstr "" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1)\n" "X-Poedit-Language: Dutch\n" #: ../../../router/java/src/net/i2p/router/Blocklist.java:126 From 8c7a39f00aba09712c2d71de6b04bff10719c5c3 Mon Sep 17 00:00:00 2001 From: zzz Date: Tue, 19 Oct 2010 14:33:11 +0000 Subject: [PATCH 05/31] * JobImpl: Deprecate getAddedBy() and addedToQueue() to reduce LogManager records --- router/java/src/net/i2p/router/JobImpl.java | 14 ++++++++++++-- router/java/src/net/i2p/router/JobQueue.java | 5 +++-- .../i2p/router/client/ClientConnectionRunner.java | 2 +- .../i2p/router/message/SendMessageDirectJob.java | 4 ++-- .../src/net/i2p/router/tunnel/pool/TestJob.java | 2 +- .../test/net/i2p/router/message/SendGarlicJob.java | 2 +- 6 files changed, 20 insertions(+), 9 deletions(-) diff --git a/router/java/src/net/i2p/router/JobImpl.java b/router/java/src/net/i2p/router/JobImpl.java index b122d3674..2f99a5357 100644 --- a/router/java/src/net/i2p/router/JobImpl.java +++ b/router/java/src/net/i2p/router/JobImpl.java @@ -39,11 +39,21 @@ public abstract class JobImpl implements Job { return buf.toString(); } + /** + * @deprecated + * As of 0.8.1, this is a noop, as it just adds classes to the log manager + * class list for no good reason. Logging in jobs is almost always + * set explicitly rather than by class name. + */ void addedToQueue() { - if (_context.logManager().getLog(getClass()).shouldLog(Log.DEBUG)) - _addedBy = new Exception(); + //if (_context.logManager().getLog(getClass()).shouldLog(Log.DEBUG)) + // _addedBy = new Exception(); } + /** + * @deprecated + * @return null always + */ public Exception getAddedBy() { return _addedBy; } public long getMadeReadyOn() { return _madeReadyOn; } public void madeReady() { _madeReadyOn = _context.clock().now(); } diff --git a/router/java/src/net/i2p/router/JobQueue.java b/router/java/src/net/i2p/router/JobQueue.java index 892a678ed..a8b5395b0 100644 --- a/router/java/src/net/i2p/router/JobQueue.java +++ b/router/java/src/net/i2p/router/JobQueue.java @@ -142,8 +142,9 @@ public class JobQueue { public void addJob(Job job) { if (job == null || !_alive) return; - if (job instanceof JobImpl) - ((JobImpl)job).addedToQueue(); + // This does nothing + //if (job instanceof JobImpl) + // ((JobImpl)job).addedToQueue(); long numReady = 0; boolean alreadyExists = false; diff --git a/router/java/src/net/i2p/router/client/ClientConnectionRunner.java b/router/java/src/net/i2p/router/client/ClientConnectionRunner.java index f363bd278..42eef3480 100644 --- a/router/java/src/net/i2p/router/client/ClientConnectionRunner.java +++ b/router/java/src/net/i2p/router/client/ClientConnectionRunner.java @@ -590,7 +590,7 @@ public class ClientConnectionRunner { + " for session [" + _sessionId.getSessionId() + "] (with nonce=2), retrying after [" + (_context.clock().now() - _lastTried) - + "]", getAddedBy()); + + "]"); } else { if (_log.shouldLog(Log.DEBUG)) _log.debug("Updating message status for message " + _messageId + " to " diff --git a/router/java/src/net/i2p/router/message/SendMessageDirectJob.java b/router/java/src/net/i2p/router/message/SendMessageDirectJob.java index be1b1e3eb..c67eeaa4a 100644 --- a/router/java/src/net/i2p/router/message/SendMessageDirectJob.java +++ b/router/java/src/net/i2p/router/message/SendMessageDirectJob.java @@ -76,7 +76,7 @@ public class SendMessageDirectJob extends JobImpl { if (_expiration < now) { if (_log.shouldLog(Log.WARN)) _log.warn("Timed out sending message " + _message + " directly (expiration = " - + new Date(_expiration) + ") to " + _targetHash.toBase64(), getAddedBy()); + + new Date(_expiration) + ") to " + _targetHash.toBase64()); if (_onFail != null) getContext().jobQueue().addJob(_onFail); return; @@ -104,7 +104,7 @@ public class SendMessageDirectJob extends JobImpl { if (_log.shouldLog(Log.WARN)) _log.warn("Unable to find the router to send to: " + _targetHash + " after searching for " + (getContext().clock().now()-_searchOn) - + "ms, message: " + _message, getAddedBy()); + + "ms, message: " + _message); if (_onFail != null) getContext().jobQueue().addJob(_onFail); } diff --git a/router/java/src/net/i2p/router/tunnel/pool/TestJob.java b/router/java/src/net/i2p/router/tunnel/pool/TestJob.java index ba3eda6d6..74910cbcb 100644 --- a/router/java/src/net/i2p/router/tunnel/pool/TestJob.java +++ b/router/java/src/net/i2p/router/tunnel/pool/TestJob.java @@ -317,7 +317,7 @@ class TestJob extends JobImpl { public String getName() { return "Tunnel test timeout"; } public void runJob() { if (_log.shouldLog(Log.WARN)) - _log.warn("Timeout: found? " + _found, getAddedBy()); + _log.warn("Timeout: found? " + _found); if (!_found) { // don't clog up the SKM with old one-tag tagsets if (_cfg.isInbound() && !_pool.getSettings().isExploratory()) { diff --git a/router/java/test/net/i2p/router/message/SendGarlicJob.java b/router/java/test/net/i2p/router/message/SendGarlicJob.java index 6b0aa0b37..f4839a522 100644 --- a/router/java/test/net/i2p/router/message/SendGarlicJob.java +++ b/router/java/test/net/i2p/router/message/SendGarlicJob.java @@ -81,7 +81,7 @@ public class SendGarlicJob extends JobImpl { long after = getContext().clock().now(); if ( (after - before) > 1000) { if (_log.shouldLog(Log.WARN)) - _log.warn("Building the garlic took too long [" + (after-before)+" ms]", getAddedBy()); + _log.warn("Building the garlic took too long [" + (after-before)+" ms]"); } else { if (_log.shouldLog(Log.DEBUG)) _log.debug("Building the garlic was fast! " + (after - before) + " ms"); From 466128c179d4d9877e9c6c8b60fcc5b7610cebdc Mon Sep 17 00:00:00 2001 From: zzz Date: Tue, 19 Oct 2010 14:39:29 +0000 Subject: [PATCH 06/31] * replaceAll() -> replace() when not using regex * ampersand escaping (lots more to do) --- .../java/src/org/klomp/snark/web/I2PSnarkServlet.java | 4 +++- apps/i2ptunnel/java/src/net/i2p/i2ptunnel/web/EditBean.java | 2 +- .../i2ptunnel/java/src/net/i2p/i2ptunnel/web/IndexBean.java | 2 +- .../java/src/net/i2p/router/web/ConfigUpdateHandler.java | 4 ++-- .../java/src/net/i2p/router/web/ConfigUpdateHelper.java | 2 +- .../java/src/net/i2p/router/web/LogsHelper.java | 6 ++++-- 6 files changed, 12 insertions(+), 8 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java index 19fb3bd0f..8a2f9a269 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java +++ b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java @@ -1214,7 +1214,9 @@ public class I2PSnarkServlet extends Default { /** @since 0.7.14 */ private static String urlify(String s) { StringBuilder buf = new StringBuilder(256); - buf.append("").append(s).append(""); + // browsers seem to work without doing this but let's be strict + String link = s.replace("&", "&"); + buf.append("").append(link).append(""); return buf.toString(); } diff --git a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/web/EditBean.java b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/web/EditBean.java index f6c76d7f6..b93b67490 100644 --- a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/web/EditBean.java +++ b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/web/EditBean.java @@ -175,7 +175,7 @@ public class EditBean extends IndexBean { } public String getAccessList(int tunnel) { - return getProperty(tunnel, "i2cp.accessList", "").replaceAll(",", "\n"); + return getProperty(tunnel, "i2cp.accessList", "").replace(",", "\n"); } public boolean getClose(int tunnel) { diff --git a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/web/IndexBean.java b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/web/IndexBean.java index d91a7900d..8559c2814 100644 --- a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/web/IndexBean.java +++ b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/web/IndexBean.java @@ -666,7 +666,7 @@ public class IndexBean { } public void setAccessList(String val) { if (val != null) - _otherOptions.put("i2cp.accessList", val.trim().replaceAll("\r\n", ",").replaceAll("\n", ",").replaceAll(" ", ",")); + _otherOptions.put("i2cp.accessList", val.trim().replace("\r\n", ",").replace("\n", ",").replace(" ", ",")); } public void setCloseTime(String val) { if (val != null) { diff --git a/apps/routerconsole/java/src/net/i2p/router/web/ConfigUpdateHandler.java b/apps/routerconsole/java/src/net/i2p/router/web/ConfigUpdateHandler.java index b407fe494..d20524bfb 100644 --- a/apps/routerconsole/java/src/net/i2p/router/web/ConfigUpdateHandler.java +++ b/apps/routerconsole/java/src/net/i2p/router/web/ConfigUpdateHandler.java @@ -142,7 +142,7 @@ public class ConfigUpdateHandler extends FormHandler { } if ( (_updateURL != null) && (_updateURL.length() > 0) ) { - _updateURL = _updateURL.replaceAll("\r\n", ",").replaceAll("\n", ","); + _updateURL = _updateURL.replace("\r\n", ",").replace("\n", ","); String oldURL = _context.router().getConfigSetting(PROP_UPDATE_URL); if ( (oldURL == null) || (!_updateURL.equals(oldURL)) ) { _context.router().setConfigSetting(PROP_UPDATE_URL, _updateURL); @@ -151,7 +151,7 @@ public class ConfigUpdateHandler extends FormHandler { } if ( (_trustedKeys != null) && (_trustedKeys.length() > 0) ) { - _trustedKeys = _trustedKeys.replaceAll("\r\n", ",").replaceAll("\n", ","); + _trustedKeys = _trustedKeys.replace("\r\n", ",").replace("\n", ","); String oldKeys = new TrustedUpdate(_context).getTrustedKeysString(); if ( (oldKeys == null) || (!_trustedKeys.equals(oldKeys)) ) { _context.router().setConfigSetting(PROP_TRUSTED_KEYS, _trustedKeys); diff --git a/apps/routerconsole/java/src/net/i2p/router/web/ConfigUpdateHelper.java b/apps/routerconsole/java/src/net/i2p/router/web/ConfigUpdateHelper.java index c9b6b7f43..520c13104 100644 --- a/apps/routerconsole/java/src/net/i2p/router/web/ConfigUpdateHelper.java +++ b/apps/routerconsole/java/src/net/i2p/router/web/ConfigUpdateHelper.java @@ -40,7 +40,7 @@ public class ConfigUpdateHelper extends HelperBase { public String getUpdateURL() { String url = _context.getProperty(ConfigUpdateHandler.PROP_UPDATE_URL); if (url != null) - return url.replaceAll(",", "\n"); + return url.replace(",", "\n"); else return ConfigUpdateHandler.DEFAULT_UPDATE_URL; } diff --git a/apps/routerconsole/java/src/net/i2p/router/web/LogsHelper.java b/apps/routerconsole/java/src/net/i2p/router/web/LogsHelper.java index f499b2ea4..9955ffeed 100644 --- a/apps/routerconsole/java/src/net/i2p/router/web/LogsHelper.java +++ b/apps/routerconsole/java/src/net/i2p/router/web/LogsHelper.java @@ -33,7 +33,7 @@ public class LogsHelper extends HelperBase { if (str == null) return ""; else { - str = str.replaceAll("<", "<").replaceAll(">", ">"); + str = str.replace("&", "&").replace("<", "<").replace(">", ">"); return _("File location") + ": " + f.getAbsolutePath() + " 
" + str + "
"; } } @@ -54,12 +54,14 @@ public class LogsHelper extends HelperBase { buf.append("\n"); for (int i = msgs.size(); i > 0; i--) { String msg = msgs.get(i - 1); + msg = msg.replace("&", "&").replace("<", "<").replace(">", ">"); buf.append("
  • "); if (colorize) { String color; // Homeland Security Advisory System // http://www.dhs.gov/xinfoshare/programs/Copy_of_press_release_0046.shtm // but pink instead of yellow for WARN + // FIXME doesnt work for translated levels if (msg.contains("CRIT")) color = "#cc0000"; else if (msg.contains("ERROR")) @@ -71,7 +73,7 @@ public class LogsHelper extends HelperBase { else color = "#006600"; buf.append(""); - buf.append(msg.replaceAll("<", "<").replaceAll(">", ">")); + buf.append(msg); buf.append(""); } else { buf.append(msg); From ed399a07d85c7929b47b32751d30fcd25eb6bc20 Mon Sep 17 00:00:00 2001 From: zzz Date: Sun, 24 Oct 2010 16:16:46 +0000 Subject: [PATCH 07/31] systray short doc and main() --- apps/systray/doc/README.txt | 21 +++++++++++++++++++ .../src/net/i2p/apps/systray/SysTray.java | 16 ++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 apps/systray/doc/README.txt diff --git a/apps/systray/doc/README.txt b/apps/systray/doc/README.txt new file mode 100644 index 000000000..3123d533f --- /dev/null +++ b/apps/systray/doc/README.txt @@ -0,0 +1,21 @@ +Bundled in ../java/lib/ are the binaries for systray4j version 2.4.1 2004-03-28, +which is still the latest. + +Files are from systray4j-2.4.1-win32.zip. + +SHA1Sums: + 28acaea97816f53d188d01fd88b72e670e67286b systray4j-2.4.1-win32.zip + a7f5e02c3652f3f1a72559e54ee69226b8b97859 systray4j.dll + 947bd91c483494256cf48ad87c211e8701b4f85b systray4j.jar + + +systray4j is GPLv2, see LICENSE.systray4j.txt. +I2P systray code in ../java/src is public domain. + +SysTray is really obsolete. It supports Windows and kde3 only. +We only instantiate it on Windows. + +The java.awt.SystemTray classes added in Java 6 +(and used by apps/desktopgui) are the way to go now. + +We could either rewrite this to use SystemTray, or switch to desktopgui. diff --git a/apps/systray/java/src/net/i2p/apps/systray/SysTray.java b/apps/systray/java/src/net/i2p/apps/systray/SysTray.java index f7e649942..23a11ef23 100644 --- a/apps/systray/java/src/net/i2p/apps/systray/SysTray.java +++ b/apps/systray/java/src/net/i2p/apps/systray/SysTray.java @@ -184,4 +184,20 @@ public class SysTray implements SysTrayMenuListener { _sysTrayMenu.addItem(_itemOpenConsole); refreshDisplay(); } + + /** + * Starts SysTray, even on linux (but requires kde3 libsystray4j.so to do anything) + * @since 0.8.1 + */ + public static void main(String args[]) { + System.err.println("SysTray4j version " + SysTrayMenu.VERSION); + System.err.println("Hit ^C to exit"); + new SysTray(); + Thread t = Thread.currentThread(); + synchronized(t) { + try { + t.wait(); + } catch (InterruptedException ie) {} + } + } } From 4bb902a8b9ea9865c3881ffae5f0b71c7a81ed9b Mon Sep 17 00:00:00 2001 From: zzz Date: Sun, 24 Oct 2010 16:49:20 +0000 Subject: [PATCH 08/31] * FileUtil: Make it easier to compile without Pack200, or with Apache Harmony's Pack200, add unzip to main() --- build.xml | 8 +++ core/java/build.xml | 3 +- core/java/src/net/i2p/util/FileUtil.java | 62 ++++++++++++++++++++++-- 3 files changed, 67 insertions(+), 6 deletions(-) diff --git a/build.xml b/build.xml index c2d26de43..3c9f08a3a 100644 --- a/build.xml +++ b/build.xml @@ -6,6 +6,14 @@ + + diff --git a/core/java/build.xml b/core/java/build.xml index a52dcb635..2ae9d655f 100644 --- a/core/java/build.xml +++ b/core/java/build.xml @@ -17,10 +17,11 @@ + - + diff --git a/core/java/src/net/i2p/util/FileUtil.java b/core/java/src/net/i2p/util/FileUtil.java index ec759e0f2..b56f197ab 100644 --- a/core/java/src/net/i2p/util/FileUtil.java +++ b/core/java/src/net/i2p/util/FileUtil.java @@ -13,10 +13,18 @@ import java.util.ArrayList; import java.util.Enumeration; import java.util.List; import java.util.jar.JarOutputStream; -import java.util.jar.Pack200; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; +// Pack200 import +// you must also uncomment the correct line in unpack() below +// For gcj, gij, etc., comment both out +// +// For Sun, OpenJDK, IcedTea, etc, use this +import java.util.jar.Pack200; + +// For Apache Harmony or if you put its pack200.jar in your library directory use this +//import org.apache.harmony.unpack200.Archive; /** * General helper methods for messing with files @@ -119,7 +127,7 @@ public class FileUtil { if (entry.getName().endsWith(".jar.pack") || entry.getName().endsWith(".war.pack")) { target = new File(targetDir, entry.getName().substring(0, entry.getName().length() - ".pack".length())); JarOutputStream fos = new JarOutputStream(new FileOutputStream(target)); - Pack200.newUnpacker().unpack(in, fos); + unpack(in, fos); fos.close(); System.err.println("INFO: File [" + entry.getName() + "] extracted and unpacked"); } else { @@ -189,9 +197,7 @@ public class FileUtil { } else { if (p200TestRequired && (entry.getName().endsWith(".jar.pack") || entry.getName().endsWith(".war.pack"))) { - try { - Class.forName("java.util.jar.Pack200", false, ClassLoader.getSystemClassLoader()); - } catch (Exception e) { // ClassNotFoundException but compiler not happy with that + if (!isPack200Supported()) { System.err.println("ERROR: Zip verify failed, your JVM does not support unpack200"); return false; } @@ -224,6 +230,40 @@ public class FileUtil { } } + /** + * This won't work right if one of the two options in unpack() is commented out. + * @since 0.8.1 + */ + private static boolean isPack200Supported() { + try { + Class.forName("java.util.jar.Pack200", false, ClassLoader.getSystemClassLoader()); + return true; + } catch (Exception e) {} + try { + Class.forName("org.apache.harmony.pack200.Archive", false, ClassLoader.getSystemClassLoader()); + return true; + } catch (Exception e) {} + return false; + } + + /** + * Caller must close streams + * @since 0.8.1 + */ + private static void unpack(InputStream in, JarOutputStream out) throws Exception { + // For Sun, OpenJDK, IcedTea, etc, use this + Pack200.newUnpacker().unpack(in, out); + + // ------------------ + // For Apache Harmony or if you put its pack200.jar in your library directory use this + //(new Archive(in, out)).unpack(); + + + // ------------------ + // For gcj, gij, etc., use this + //throw new IOException("Pack200 not supported"); + } + /** * Read in the last few lines of a (newline delimited) textfile, or null if * the file doesn't exist. @@ -352,6 +392,18 @@ public class FileUtil { boolean copied = FileUtil.copy(args[1], args[2], false); if (!copied) System.err.println("Error copying [" + args[1] + "] to [" + args[2] + "]"); + } else if ("unzip".equals(args[0])) { + File f = new File(args[1]); + File to = new File("tmp"); + to.mkdir(); + boolean copied = verifyZip(f); + if (!copied) + System.err.println("Error verifying " + args[1]); + copied = extractZip(f, to); + if (copied) + System.err.println("Unzipped [" + args[1] + "] to [" + to + "]"); + else + System.err.println("Error unzipping [" + args[1] + "] to [" + to + "]"); } } From b1f1725506f6a0938472a4771e8f6b80ce13fc8f Mon Sep 17 00:00:00 2001 From: zzz Date: Sun, 24 Oct 2010 16:52:41 +0000 Subject: [PATCH 09/31] * Router: Set permissions on wrapper.log when not called by RouterLaunch --- .../src/net/i2p/util/SecureFileOutputStream.java | 2 +- router/java/src/net/i2p/router/Router.java | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/core/java/src/net/i2p/util/SecureFileOutputStream.java b/core/java/src/net/i2p/util/SecureFileOutputStream.java index 8a891344b..3df1f14a5 100644 --- a/core/java/src/net/i2p/util/SecureFileOutputStream.java +++ b/core/java/src/net/i2p/util/SecureFileOutputStream.java @@ -56,7 +56,7 @@ public class SecureFileOutputStream extends FileOutputStream { * Tries to set the permissions to 600, * ignores errors */ - private static void setPerms(File f) { + public static void setPerms(File f) { if (!canSetPerms) return; try { diff --git a/router/java/src/net/i2p/router/Router.java b/router/java/src/net/i2p/router/Router.java index a091cba13..bfcbe83a6 100644 --- a/router/java/src/net/i2p/router/Router.java +++ b/router/java/src/net/i2p/router/Router.java @@ -218,6 +218,19 @@ public class Router { // NOW we start all the activity _context.initAll(); + // Set wrapper.log permissions. + // Just hope this is the right location, we don't know for sure, + // but this is the same method used in LogsHelper and we have no complaints. + // (we could look for the wrapper.config file and parse it I guess...) + // If we don't have a wrapper, RouterLaunch does this for us. + if (System.getProperty("wrapper.version") != null) { + File f = new File(System.getProperty("java.io.tmpdir"), "wrapper.log"); + if (!f.exists()) + f = new File(_context.getBaseDir(), "wrapper.log"); + if (f.exists()) + SecureFileOutputStream.setPerms(f); + } + _routerInfo = null; _higherVersionSeen = false; _log = _context.logManager().getLog(Router.class); From adab0cc3d396141d8a2d818c57e2d1398fc209b1 Mon Sep 17 00:00:00 2001 From: zzz Date: Sun, 24 Oct 2010 16:55:29 +0000 Subject: [PATCH 10/31] * NTCP: Catch a race after stop() --- .../router/transport/ntcp/NTCPSendFinisher.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/router/java/src/net/i2p/router/transport/ntcp/NTCPSendFinisher.java b/router/java/src/net/i2p/router/transport/ntcp/NTCPSendFinisher.java index 207db4c3c..fd5cf1ac9 100644 --- a/router/java/src/net/i2p/router/transport/ntcp/NTCPSendFinisher.java +++ b/router/java/src/net/i2p/router/transport/ntcp/NTCPSendFinisher.java @@ -2,6 +2,7 @@ package net.i2p.router.transport.ntcp; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.ThreadFactory; @@ -24,9 +25,9 @@ import net.i2p.util.Log; */ public class NTCPSendFinisher { private static final int THREADS = 4; - private I2PAppContext _context; - private NTCPTransport _transport; - private Log _log; + private final I2PAppContext _context; + private final NTCPTransport _transport; + private final Log _log; private int _count; private ThreadPoolExecutor _executor; @@ -47,7 +48,12 @@ public class NTCPSendFinisher { } public void add(OutNetMessage msg) { - _executor.execute(new RunnableEvent(msg)); + try { + _executor.execute(new RunnableEvent(msg)); + } catch (RejectedExecutionException ree) { + // race with stop() + _log.warn("NTCP send finisher stopped, discarding msg.afterSend()"); + } } // not really needed for now but in case we want to add some hooks like afterExecute() From 4f9c442d556619fe5344a7b592e60068fc952447 Mon Sep 17 00:00:00 2001 From: zzz Date: Sun, 24 Oct 2010 16:56:43 +0000 Subject: [PATCH 11/31] fix disconnect race NPE --- apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java index 195659c50..3c1cad535 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java +++ b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java @@ -190,6 +190,7 @@ public class I2PSnarkUtil { */ public void disconnect() { I2PSocketManager mgr = _manager; + // FIXME this can cause race NPEs elsewhere _manager = null; _shitlist.clear(); mgr.destroySocketManager(); @@ -201,6 +202,9 @@ public class I2PSnarkUtil { /** connect to the given destination */ I2PSocket connect(PeerID peer) throws IOException { + I2PSocketManager mgr = _manager; + if (mgr == null) + throw new IOException("No socket manager"); Destination addr = peer.getAddress(); if (addr == null) throw new IOException("Null address"); From ac3e6e27dc0596da976f97f7fb260fe80f148475 Mon Sep 17 00:00:00 2001 From: zzz Date: Sun, 24 Oct 2010 16:57:38 +0000 Subject: [PATCH 12/31] add note --- .../java/src/net/i2p/client/streaming/ConnectionOptions.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/streaming/java/src/net/i2p/client/streaming/ConnectionOptions.java b/apps/streaming/java/src/net/i2p/client/streaming/ConnectionOptions.java index c6596b7c9..72eb38e60 100644 --- a/apps/streaming/java/src/net/i2p/client/streaming/ConnectionOptions.java +++ b/apps/streaming/java/src/net/i2p/client/streaming/ConnectionOptions.java @@ -47,6 +47,9 @@ public class ConnectionOptions extends I2PSocketOptionsImpl { private int _maxTotalConnsPerHour; private int _maxTotalConnsPerDay; + // NOTE - almost all the options are below, but see + // I2PSocketOptions in ministreaming for a few more + public static final int PROFILE_BULK = 1; public static final int PROFILE_INTERACTIVE = 2; From f239d4f350e92e3af5c4ec2aa5498b74c8da81fa Mon Sep 17 00:00:00 2001 From: complication Date: Tue, 26 Oct 2010 23:38:01 +0000 Subject: [PATCH 13/31] * Tiny readability fix: separate value and unit with a space. --- .../java/src/net/i2p/router/web/TunnelRenderer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/routerconsole/java/src/net/i2p/router/web/TunnelRenderer.java b/apps/routerconsole/java/src/net/i2p/router/web/TunnelRenderer.java index 9206340ca..e9d7d1da6 100644 --- a/apps/routerconsole/java/src/net/i2p/router/web/TunnelRenderer.java +++ b/apps/routerconsole/java/src/net/i2p/router/web/TunnelRenderer.java @@ -112,7 +112,7 @@ public class TunnelRenderer { if (lifetime > 10*60) lifetime = 10*60; int bps = 1024 * (int) cfg.getProcessedMessagesCount() / lifetime; - out.write("" + bps + "Bps"); + out.write("" + bps + " Bps"); if (cfg.getSendTo() == null) out.write("" + _("Outbound Endpoint") + ""); else if (cfg.getReceiveFrom() == null) From b9af4a8cf0a6f8b3faaf08bbcd8ba0a05a28c41f Mon Sep 17 00:00:00 2001 From: zzz Date: Wed, 27 Oct 2010 13:25:13 +0000 Subject: [PATCH 14/31] avoid rare transport NPE at startup --- .../src/net/i2p/router/transport/CommSystemFacadeImpl.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/router/java/src/net/i2p/router/transport/CommSystemFacadeImpl.java b/router/java/src/net/i2p/router/transport/CommSystemFacadeImpl.java index 196876383..39856f509 100644 --- a/router/java/src/net/i2p/router/transport/CommSystemFacadeImpl.java +++ b/router/java/src/net/i2p/router/transport/CommSystemFacadeImpl.java @@ -136,17 +136,17 @@ public class CommSystemFacadeImpl extends CommSystemFacade { @Override public boolean isBacklogged(Hash dest) { - return _manager.isBacklogged(dest); + return _manager != null && _manager.isBacklogged(dest); } @Override public boolean isEstablished(Hash dest) { - return _manager.isEstablished(dest); + return _manager != null && _manager.isEstablished(dest); } @Override public boolean wasUnreachable(Hash dest) { - return _manager.wasUnreachable(dest); + return _manager != null && _manager.wasUnreachable(dest); } @Override From 983e7683fdd59f60c3462a2c613ff02a8b94c8f8 Mon Sep 17 00:00:00 2001 From: zzz Date: Wed, 27 Oct 2010 13:29:27 +0000 Subject: [PATCH 15/31] * i2psnark: - Don't stay interested if we run out of pieces to request (thanks sponge) - Enhance debug mode to show requests --- .../java/src/org/klomp/snark/Peer.java | 7 +++- .../java/src/org/klomp/snark/PeerState.java | 37 ++++++++++++++----- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/Peer.java b/apps/i2psnark/java/src/org/klomp/snark/Peer.java index da1003aab..2120b5c2e 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Peer.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Peer.java @@ -117,10 +117,15 @@ public class Peer implements Comparable } /** - * Returns socket (for debug printing) + * @return socket debug string (for debug printing) */ public String getSocket() { + if (state != null) { + String r = state.getRequests(); + if (r != null) + return sock.toString() + "
    Requests: " + r; + } return sock.toString(); } diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerState.java b/apps/i2psnark/java/src/org/klomp/snark/PeerState.java index 5e9250e22..1dd40c3cf 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerState.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerState.java @@ -58,9 +58,6 @@ class PeerState private final List outstandingRequests = new ArrayList(); private Request lastRequest = null; - // If we have te resend outstanding requests (true after we got choked). - private boolean resend = false; - private final static int MAX_PIPELINE = 5; // this is for outbound requests private final static int MAX_PIPELINE_BYTES = 128*1024; // this is for inbound requests public final static int PARTSIZE = 16*1024; // outbound request @@ -91,14 +88,13 @@ class PeerState if (_log.shouldLog(Log.DEBUG)) _log.debug(peer + " rcv " + (choke ? "" : "un") + "choked"); + boolean resend = choked && !choke; choked = choke; - if (choked) - resend = true; listener.gotChoke(peer, choke); - if (!choked && interesting) - request(); + if (interesting && !choked) + request(resend); } void interestedMessage(boolean interest) @@ -464,7 +460,7 @@ class PeerState } // Starts or resumes requesting pieces. - private void request() + private void request(boolean resend) { // Are there outstanding requests that have to be resend? if (resend) @@ -472,7 +468,6 @@ class PeerState synchronized (this) { out.sendRequests(outstandingRequests); } - resend = false; } // Add/Send some more requests if necessary. @@ -526,6 +521,7 @@ class PeerState /** * Starts requesting first chunk of next piece. Returns true if * something has been added to the requests, false otherwise. + * Caller should synchronize. */ private boolean requestNextPiece() { @@ -587,6 +583,15 @@ class PeerState } } + // If we are not in the end game, we may run out of things to request + // because we are asking other peers. Set not-interesting now rather than + // wait for those other requests to be satisfied via havePiece() + if (interesting && lastRequest == null) { + interesting = false; + out.sendInterest(false); + if (_log.shouldLog(Log.DEBUG)) + _log.debug(peer + " nothing more to request, now uninteresting"); + } return false; } @@ -601,7 +606,7 @@ class PeerState out.sendInterest(interest); if (interesting && !choked) - request(); + request(true); // we shouldnt have any pending requests, but if we do, resend them } } @@ -627,4 +632,16 @@ class PeerState if (interesting && !choked) out.retransmitRequests(outstandingRequests); } + + /** + * debug + * @return string or null + * @since 0.8.1 + */ + synchronized String getRequests() { + if (outstandingRequests.isEmpty()) + return null; + else + return outstandingRequests.toString(); + } } From 571ad83e03144ec1aac5abd2f2d7b74adccff040 Mon Sep 17 00:00:00 2001 From: zzz Date: Wed, 27 Oct 2010 14:22:53 +0000 Subject: [PATCH 16/31] use nbsp in formatSize2() --- core/java/src/net/i2p/data/DataHelper.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/core/java/src/net/i2p/data/DataHelper.java b/core/java/src/net/i2p/data/DataHelper.java index bb1ffe55c..c0fa56e3a 100644 --- a/core/java/src/net/i2p/data/DataHelper.java +++ b/core/java/src/net/i2p/data/DataHelper.java @@ -1076,7 +1076,8 @@ public class DataHelper { } /** - * Like formatSize but with a space after the number + * Like formatSize but with a non-breaking space after the number + * Use only in HTML * @since 0.7.14 */ public static String formatSize2(long bytes) { @@ -1091,11 +1092,11 @@ public class DataHelper { String str = fmt.format(val); switch (scale) { - case 1: return str + " K"; - case 2: return str + " M"; - case 3: return str + " G"; - case 4: return str + " T"; - default: return bytes + " "; + case 1: return str + " K"; + case 2: return str + " M"; + case 3: return str + " G"; + case 4: return str + " T"; + default: return bytes + " "; } } From 7efb0fa7ed858b98f11ac769cd617f2c2671b598 Mon Sep 17 00:00:00 2001 From: zzz Date: Fri, 29 Oct 2010 17:32:05 +0000 Subject: [PATCH 17/31] * i2psnark: - Priority mapping bugfix - Close files as we go when creating/checking so we don't run out of file descriptors --- .../java/src/org/klomp/snark/Storage.java | 41 +++++++++++++++---- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/Storage.java b/apps/i2psnark/java/src/org/klomp/snark/Storage.java index d208adc34..e77d04432 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Storage.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Storage.java @@ -422,7 +422,7 @@ public class Storage file++; long oldFileEnd = fileEnd; fileEnd += lengths[file]; - if (priorities[file] > pri && pcEnd < oldFileEnd) + if (priorities[file] > pri && oldFileEnd < pcEnd) pri = priorities[file]; } rv[i] = pri; @@ -669,6 +669,10 @@ public class Storage changed = true; synchronized(RAFlock[i]) { allocateFile(i); + // close as we go so we don't run out of file descriptors + try { + closeRAF(i); + } catch (IOException ioe) {} } } else { _util.debug("File '" + names[i] + "' exists, but has wrong length - repairing corruption", Snark.ERROR); @@ -677,8 +681,10 @@ public class Storage synchronized(RAFlock[i]) { checkRAF(i); rafs[i].setLength(lengths[i]); + try { + closeRAF(i); + } catch (IOException ioe) {} } - // will be closed below } } @@ -687,10 +693,25 @@ public class Storage { pieces = metainfo.getPieces(); byte[] piece = new byte[metainfo.getPieceLength(0)]; + int file = 0; + long fileEnd = lengths[0]; + long pieceEnd = 0; for (int i = 0; i < pieces; i++) { int length = getUncheckedPiece(i, piece); boolean correctHash = metainfo.checkPiece(i, piece, 0, length); + // close as we go so we don't run out of file descriptors + pieceEnd += length; + while (fileEnd <= pieceEnd) { + synchronized(RAFlock[file]) { + try { + closeRAF(file); + } catch (IOException ioe) {} + } + if (++file >= rafs.length) + break; + fileEnd += lengths[file]; + } if (correctHash) { bitfield.set(i); @@ -705,13 +726,14 @@ public class Storage _probablyComplete = complete(); // close all the files so we don't end up with a zillion open ones; // we will reopen as needed - for (int i = 0; i < rafs.length; i++) { - synchronized(RAFlock[i]) { - try { - closeRAF(i); - } catch (IOException ioe) {} - } - } + // Now closed above to avoid running out of file descriptors + //for (int i = 0; i < rafs.length; i++) { + // synchronized(RAFlock[i]) { + // try { + // closeRAF(i); + // } catch (IOException ioe) {} + // } + //} if (listener != null) { listener.storageAllChecked(this); @@ -720,6 +742,7 @@ public class Storage } } + /** this calls openRAF(); caller must synnchronize and call closeRAF() */ private void allocateFile(int nr) throws IOException { // caller synchronized From 9baa6e7bc8d147e781b3b2fbffb3b89321983473 Mon Sep 17 00:00:00 2001 From: zzz Date: Fri, 29 Oct 2010 20:31:07 +0000 Subject: [PATCH 18/31] update requests after changing priorities --- .../java/src/org/klomp/snark/Peer.java | 22 ++++++ .../src/org/klomp/snark/PeerCoordinator.java | 31 ++++++-- .../java/src/org/klomp/snark/PeerState.java | 74 +++++++++++-------- 3 files changed, 92 insertions(+), 35 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/Peer.java b/apps/i2psnark/java/src/org/klomp/snark/Peer.java index 2120b5c2e..0445ab205 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Peer.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Peer.java @@ -392,6 +392,28 @@ public class Peer implements Comparable s.havePiece(piece); } + /** + * Tell the other side that we are no longer interested in any of + * the outstanding requests (if any) for this piece. + * @since 0.8.1 + */ + void cancel(int piece) { + PeerState s = state; + if (s != null) + s.cancelPiece(piece); + } + + /** + * Update the request queue. + * Call after adding wanted pieces. + * @since 0.8.1 + */ + void request() { + PeerState s = state; + if (s != null) + s.addRequest(); + } + /** * Whether or not the peer is interested in pieces we have. Returns * false if not connected. diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java index 07c832440..6e7956078 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java @@ -460,7 +460,7 @@ public class PeerCoordinator implements PeerListener } /** - * Returns true if we don't have the given piece yet. + * @return true if we still want the given piece */ public boolean gotHave(Peer peer, int piece) { @@ -586,8 +586,10 @@ public class PeerCoordinator implements PeerListener */ public void updatePiecePriorities() { int[] pri = storage.getPiecePriorities(); - if (pri == null) + if (pri == null) { + _log.debug("Updated piece priorities called but no priorities to set?"); return; + } synchronized(wantedPieces) { // Add incomplete and previously unwanted pieces to the list // Temp to avoid O(n**2) @@ -619,14 +621,31 @@ public class PeerCoordinator implements PeerListener // now set the new priorities and remove newly unwanted pieces for (Iterator iter = wantedPieces.iterator(); iter.hasNext(); ) { Piece p = iter.next(); - int id = pri[p.getId()]; - if (id >= 0) - p.setPriority(pri[p.getId()]); - else + int priority = pri[p.getId()]; + if (priority >= 0) { + p.setPriority(priority); + } else { iter.remove(); + // cancel all peers + synchronized(peers) { + for (Peer peer : peers) { + peer.cancel(p.getId()); + } + } + } } + if (_log.shouldLog(Log.DEBUG)) + _log.debug("Updated piece priorities, now wanted: " + wantedPieces); // if we added pieces, they will be in-order unless we shuffle Collections.shuffle(wantedPieces, _random); + + // update request queues, in case we added wanted pieces + // and we were previously uninterested + synchronized(peers) { + for (Peer peer : peers) { + peer.request(); + } + } } } diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerState.java b/apps/i2psnark/java/src/org/klomp/snark/PeerState.java index 1dd40c3cf..cfa1575af 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerState.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerState.java @@ -55,7 +55,7 @@ class PeerState final PeerConnectionOut out; // Outstanding request - private final List outstandingRequests = new ArrayList(); + private final List outstandingRequests = new ArrayList(); private Request lastRequest = null; private final static int MAX_PIPELINE = 5; // this is for outbound requests @@ -274,7 +274,7 @@ class PeerState synchronized private int getFirstOutstandingRequest(int piece) { for (int i = 0; i < outstandingRequests.size(); i++) - if (((Request)outstandingRequests.get(i)).piece == piece) + if (outstandingRequests.get(i).piece == piece) return i; return -1; } @@ -309,12 +309,12 @@ class PeerState Request req; synchronized(this) { - req = (Request)outstandingRequests.get(r); + req = outstandingRequests.get(r); while (req.piece == piece && req.off != begin && r < outstandingRequests.size() - 1) { r++; - req = (Request)outstandingRequests.get(r); + req = outstandingRequests.get(r); } // Something wrong? @@ -338,7 +338,7 @@ class PeerState + ", wanted for peer: " + peer); for (int i = 0; i < r; i++) { - Request dropReq = (Request)outstandingRequests.remove(0); + Request dropReq = outstandingRequests.remove(0); outstandingRequests.add(dropReq); if (!choked) out.sendRequest(dropReq); @@ -362,11 +362,11 @@ class PeerState { Request req = null; for (int i = 0; i < outstandingRequests.size(); i++) { - Request r1 = (Request)outstandingRequests.get(i); + Request r1 = outstandingRequests.get(i); int j = getFirstOutstandingRequest(r1.piece); if (j == -1) continue; - Request r2 = (Request)outstandingRequests.get(j); + Request r2 = outstandingRequests.get(j); if (r2.off > 0 && ((req == null) || (r2.off > req.off))) req = r2; } @@ -394,7 +394,7 @@ class PeerState } Request req = null; for (int i = 0; i < size; i++) { - Request r1 = (Request)outstandingRequests.get(i); + Request r1 = outstandingRequests.get(i); if (pc != r1.piece) { pc = r1.piece; arr[pos++] = pc; @@ -419,32 +419,19 @@ class PeerState + " length: " + bs.length); } + /** + * We now have this piece. + * Tell the peer and cancel any requests for the piece. + */ void havePiece(int piece) { if (_log.shouldLog(Log.DEBUG)) _log.debug("Tell " + peer + " havePiece(" + piece + ")"); - synchronized(this) - { // Tell the other side that we are no longer interested in any of // the outstanding requests for this piece. - if (lastRequest != null && lastRequest.piece == piece) - lastRequest = null; - - Iterator it = outstandingRequests.iterator(); - while (it.hasNext()) - { - Request req = (Request)it.next(); - if (req.piece == piece) - { - it.remove(); - // Send cancel even when we are choked to make sure that it is - // really never ever send. - out.sendCancel(req); - } - } - } - + cancelPiece(piece); + // Tell the other side that we really have this piece. out.sendHave(piece); @@ -459,7 +446,33 @@ class PeerState } } - // Starts or resumes requesting pieces. + /** + * Tell the other side that we are no longer interested in any of + * the outstanding requests (if any) for this piece. + * @since 0.8.1 + */ + synchronized void cancelPiece(int piece) { + if (lastRequest != null && lastRequest.piece == piece) + lastRequest = null; + + Iterator it = outstandingRequests.iterator(); + while (it.hasNext()) + { + Request req = it.next(); + if (req.piece == piece) + { + it.remove(); + // Send cancel even when we are choked to make sure that it is + // really never ever send. + out.sendCancel(req); + } + } + } + + /** + * Starts or resumes requesting pieces. + * @param resend should we resend outstanding requests? + */ private void request(boolean resend) { // Are there outstanding requests that have to be resend? @@ -476,8 +489,11 @@ class PeerState /** * Adds a new request to the outstanding requests list. + * Then send interested if we weren't. + * Then send new requests if not choked. + * If nothing to request, send not interested if we were. */ - synchronized private void addRequest() + synchronized void addRequest() { boolean more_pieces = true; while (more_pieces) From 502cf72653b8e0361ae8b25eb29dc38b36ea6007 Mon Sep 17 00:00:00 2001 From: zzz Date: Sat, 30 Oct 2010 00:38:08 +0000 Subject: [PATCH 19/31] fix nbsp screwing up POST --- .../java/src/net/i2p/router/web/ConfigLoggingHelper.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/routerconsole/java/src/net/i2p/router/web/ConfigLoggingHelper.java b/apps/routerconsole/java/src/net/i2p/router/web/ConfigLoggingHelper.java index ce41b1e12..f316e7dc6 100644 --- a/apps/routerconsole/java/src/net/i2p/router/web/ConfigLoggingHelper.java +++ b/apps/routerconsole/java/src/net/i2p/router/web/ConfigLoggingHelper.java @@ -24,7 +24,9 @@ public class ConfigLoggingHelper extends HelperBase { public String getMaxFileSize() { int bytes = _context.logManager().getFileSize(); if (bytes <= 0) return "1.00 MB"; - return DataHelper.formatSize2(bytes) + 'B'; + // " " comes back in the POST as 0xc2 0xa0 + // which is not recognized as whitespace and who knows why + return DataHelper.formatSize2(bytes).replace(" ", " ") + 'B'; } public String getLogLevelTable() { StringBuilder buf = new StringBuilder(32*1024); From c76058efc36e7d56b877d3ffa323e3cece90619d Mon Sep 17 00:00:00 2001 From: zzz Date: Sat, 30 Oct 2010 14:17:09 +0000 Subject: [PATCH 20/31] send &compact=1 to keep opentracker happy --- apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java b/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java index d904f98ca..331bd770b 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java +++ b/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java @@ -357,7 +357,7 @@ public class TrackerClient extends I2PAppThread + "&uploaded=" + uploaded + "&downloaded=" + downloaded + "&left=" + left - + "&compact" + + "&compact=1" // NOTE: opentracker will return 400 for &compact alone + ((! event.equals(NO_EVENT)) ? ("&event=" + event) : ""); if (left <= 0 || event.equals(STOPPED_EVENT) || !coordinator.needPeers()) s += "&numwant=0"; From 643687472aff141c253265e8f50aa872c1b85581 Mon Sep 17 00:00:00 2001 From: zzz Date: Sat, 30 Oct 2010 15:28:29 +0000 Subject: [PATCH 21/31] - Only add wanted pieces to wanted list at startup - Make sure lastRequest is null when it should be - Logging tweaks --- .../src/org/klomp/snark/PeerCoordinator.java | 6 +++++- .../java/src/org/klomp/snark/PeerState.java | 19 +++++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java index 6e7956078..0fa74e954 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java @@ -105,11 +105,15 @@ public class PeerCoordinator implements PeerListener public void setWantedPieces() { // Make a list of pieces + // FIXME synchronize, clear and re-add instead? + // Don't replace something we are synchronizing on. wantedPieces = new ArrayList(); BitField bitfield = storage.getBitField(); int[] pri = storage.getPiecePriorities(); for(int i = 0; i < metainfo.getPieces(); i++) { - if (!bitfield.get(i)) { + // only add if we don't have and the priority is >= 0 + if ((!bitfield.get(i)) && + (pri == null || pri[i] >= 0)) { Piece p = new Piece(i); if (pri != null) p.setPriority(pri[i]); diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerState.java b/apps/i2psnark/java/src/org/klomp/snark/PeerState.java index cfa1575af..cadcbf866 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerState.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerState.java @@ -56,6 +56,7 @@ class PeerState // Outstanding request private final List outstandingRequests = new ArrayList(); + /** the tail (NOT the head) of the request queue */ private Request lastRequest = null; private final static int MAX_PIPELINE = 5; // this is for outbound requests @@ -565,11 +566,10 @@ class PeerState } } int nextPiece = listener.wantPiece(peer, bitfield); - if (_log.shouldLog(Log.DEBUG)) - _log.debug(peer + " want piece " + nextPiece); - if (nextPiece != -1 - && (lastRequest == null || lastRequest.piece != nextPiece)) - { + if (nextPiece != -1 + && (lastRequest == null || lastRequest.piece != nextPiece)) { + if (_log.shouldLog(Log.DEBUG)) + _log.debug(peer + " want piece " + nextPiece); // Fail safe to make sure we are interested // When we transition into the end game we may not be interested... if (!interesting) { @@ -596,9 +596,16 @@ class PeerState out.sendRequest(req); lastRequest = req; return true; - } + } else { + if (_log.shouldLog(Log.DEBUG)) + _log.debug(peer + " no more pieces to request"); + } } + // failsafe + if (outstandingRequests.isEmpty()) + lastRequest = null; + // If we are not in the end game, we may run out of things to request // because we are asking other peers. Set not-interesting now rather than // wait for those other requests to be satisfied via havePiece() From f170baab3f97cbbbfeaf15198007b191a8f48416 Mon Sep 17 00:00:00 2001 From: zzz Date: Sat, 30 Oct 2010 15:30:14 +0000 Subject: [PATCH 22/31] - Delay during StopAll so we don't close the tunnel before the stopped announces go out and reopen it - Logging tweaks --- apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java | 3 +++ apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java | 2 +- .../i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java index 3c1cad535..8edc92fdf 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java +++ b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java @@ -152,6 +152,9 @@ public class I2PSnarkUtil { */ synchronized public boolean connect() { if (_manager == null) { + // try to find why reconnecting after stop + if (_log.shouldLog(Log.DEBUG)) + _log.debug("Connecting to I2P", new Exception("I did it")); Properties opts = new Properties(); if (_opts != null) { for (Iterator iter = _opts.keySet().iterator(); iter.hasNext(); ) { diff --git a/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java b/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java index 331bd770b..e3cc8ae9f 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java +++ b/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java @@ -268,7 +268,7 @@ public class TrackerClient extends I2PAppThread List ordered = new ArrayList(peers); Collections.shuffle(ordered, r); Iterator it = ordered.iterator(); - while (it.hasNext()) { + while ((!stop) && it.hasNext()) { Peer cur = (Peer)it.next(); // FIXME if id == us || dest == us continue; // only delay if we actually make an attempt to add peer diff --git a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java index 8a2f9a269..e284cc172 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java +++ b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java @@ -530,6 +530,8 @@ public class I2PSnarkServlet extends Default { _manager.stopTorrent(snark.torrent, false); } if (_manager.util().connected()) { + // Give the stopped announces time to get out + try { Thread.sleep(2000); } catch (InterruptedException ie) {} _manager.util().disconnect(); _manager.addMessage(_("I2P tunnel closed.")); } From 30a5c4907b80aad4988b10e6f907eff667d4e585 Mon Sep 17 00:00:00 2001 From: zzz Date: Sat, 30 Oct 2010 15:50:03 +0000 Subject: [PATCH 23/31] -6; history for this head --- history.txt | 139 ++++++++++++++++++ .../src/net/i2p/router/RouterVersion.java | 2 +- 2 files changed, 140 insertions(+), 1 deletion(-) diff --git a/history.txt b/history.txt index 78e36b4bb..ab77d569f 100644 --- a/history.txt +++ b/history.txt @@ -1,3 +1,142 @@ +2010-10-30 zzz + * i2psnark: + - Priority mapping bugfix + - Close files as we go when creating/checking + so we don't run out of file descriptors + - Update request queues after priority change + - Only add wanted pieces to wanted list at startup + - Make sure lastRequest is null when it should be + - Delay during StopAll so we don't close the tunnel before the + stopped announces go out and reopen it + - Logging tweaks + +2010-10-27 zzz + * i2psnark: + - Don't stay interested if we run out of pieces + to request (thanks sponge) + - Enhance debug mode to show requests + - Priority mapping bugfix + * Transport: Avoid rare NPE at startup + +2010-10-24 zzz + * FileUtil: Make it easier to compile without Pack200, or with + Apache Harmony's Pack200, add unzip to main() + * i2psnark: Catch a race after disconnect() + * NTCP: Catch a race after stop() + * Router: Set permissions on wrapper.log when not called by RouterLaunch + * Systray: New doc and main() + +2010-10-19 zzz + * Escape & in logs and i2psnark (much more to do) + * JobImpl: Deprecate two debugging methods + * replaceAll() -> replace() when we don't need regex + +2010-10-15 zzz + * i2psnark: Add file priority feature + * I2PSocketEepGet: Set connect delay to save a RTT, will + speed announces in i2psnark + +2010-10-12 zzz + *** 1.6 or higher JDK now required to build + * configlogging.jsp: + - Add easy way to add an override + - Make file size specifier more flexible + * Console: + - Sort RouterAddress options on netdb.jsp and peers.jsp + - Remove unused web-*.xml file from war + * Crypto: + - Convert all ArrayList caching to LBQs in YKGenerator, + HMACGenerator, and AESKeyCache. + - Change DSAEngine params from Hash to new SHA1Hash, since + these were really 20 byte hashes, not 32 byte Hashes. + - Add stats to track YKGenerator caching success + - Fix YKGenerator precalculation to be much more useful by + increasing the cache size and dramatically shortening the delay + - Option cleanups + - YKGenerator cleanups + - Mark HMAC256Generator unused + * EepGet: Reset length variable on redirect + * Files: Change permissions to 600/700 for all written files/directories. + Now requires Java 1.6 to build, but only 1.5+ to run. + (requires 1.6 to set permissiomns) + * GeoIP: Fix locking bug causing lookups to stop + * Hash: Throw IAE if data length is not 32 bytes, + now that DSAEngine abuse is gone + * HTTPResponseOutputStream: + - More caching + - Stats cleanup + - Max header length check + - Catch OOM + - Initializer cleanup + - Javadoc + * I2CP: + - Add new option i2cp.messageReliability=none, which prevents the + router from sending MessageStatusMessages back in reply to an + outbound SendMessageMessage. Since the streaming lib always ignored + the MSMs anyway, make it the default for streaming. + This will reduce the I2CP traffic significantly. + MSM handling now avoided, but it is still fairly broken, see + comments in I2PSessionImpl2. + - Cleanups to replace method calls with fields + - More cleanups, javadoc, rate reduction + * i2psnark: + - Compact response format + - Add link to finished torrent in message box + - Don't let one bad torrent prevent others from + starting or stopping + - Sort peers by completion % + - Add some missing mime types to web.xml + - shouldLog() cleanup + * i2ptunnel: + - Now that streaming flush() is fixed, use it in IRCClient, and + for initial data in I2PTunnel runner, to avoid the 250 ms + passive flush delay + - Add hostname DSA signature field, to be used for addkey forms. + Experimental, may be commented out later. + - More header blocking (thanks telecomix!) + - Remove unused web-*.xml file from war + * Installer: Add startup hint for non-x86 + * Javadoc updates all over the place + * LogConsoleBuffer: Java 5 + * Naming: + - Increase cache size and expiration time + - Add clearCache() method + - Don't use EepGet or Exec for b32 + - Javadoc updates + * NetDB: + - Expire unreachable routers quickly, even if they don't have introducers, + so we don't have old data on routers that ran out of introducers. + - Fix rare NPEs at shutdown + * NTCP: + - Cleanups + * Streaming: + - Make flush() block less, by waiting only for "accept" into the + streaming queue rather than "completion" (i.e. ACK from the far end). + This prevents complete stalls when flushing, and should help performance + of apps that use flush(), like i2psnark (and SAM?). + close() still does a flush that waits for completion, as i2ptunnel + doesn't like a fast return from close(). + - cleanups + * SusiDNS: + - Remove unused web-*.xml file from war + * TransportManager: Convert _transports from a List to a CHM + to prevent a rare concurrent exception + * Tunnels: + - Don't use peers < 0.7.9 for tunnels due to the old + message corruption bugs + - Javadoc + - Cleanups + * UDP: + - Beginnings of destroy message support + - Try to avoid running out of introducers by relaxing selection criteria + and increasing minimum number of potential introducers + - Avoid rare AIOOBE + - PacketBuilder refactor + - Make most classes package private + - Comments + - Logging cleanup + - Comment out a main() + * 2010-07-12 0.8 released 2010-07-08 zzz diff --git a/router/java/src/net/i2p/router/RouterVersion.java b/router/java/src/net/i2p/router/RouterVersion.java index a411b0099..c10128fe6 100644 --- a/router/java/src/net/i2p/router/RouterVersion.java +++ b/router/java/src/net/i2p/router/RouterVersion.java @@ -18,7 +18,7 @@ public class RouterVersion { /** deprecated */ public final static String ID = "Monotone"; public final static String VERSION = CoreVersion.VERSION; - public final static long BUILD = 0; + public final static long BUILD = 6; /** for example "-test" */ public final static String EXTRA = ""; From 2ea3f9b9bb7b99b8f6b8509d736ed7e6531994e0 Mon Sep 17 00:00:00 2001 From: zzz Date: Sun, 31 Oct 2010 14:33:31 +0000 Subject: [PATCH 24/31] parse log limit with current locale (ticket 118) --- core/java/src/net/i2p/util/LogManager.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/java/src/net/i2p/util/LogManager.java b/core/java/src/net/i2p/util/LogManager.java index d3ece13c5..634b4c1ad 100644 --- a/core/java/src/net/i2p/util/LogManager.java +++ b/core/java/src/net/i2p/util/LogManager.java @@ -13,6 +13,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.text.DecimalFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.HashMap; @@ -429,7 +430,8 @@ public class LogManager { v = v.substring(0, v.length() - 1); char mod = v.charAt(v.length() - 1); if (!Character.isDigit(mod)) v = v.substring(0, v.length() - 1); - double val = Double.parseDouble(v); + // output to form was in current locale, so have to parse it back that way + double val = (new DecimalFormat()).parse(v.trim()).doubleValue(); switch (mod) { case 'K': val *= 1024; From 0afabbd609fc057a7dcb2a8db2ee8622ccd55832 Mon Sep 17 00:00:00 2001 From: zzz Date: Sun, 31 Oct 2010 14:36:56 +0000 Subject: [PATCH 25/31] Add synch to fix race causing AIOOBE http://forum.i2p/viewtopic.php?t=5061 --- .../router/client/ClientConnectionRunner.java | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/router/java/src/net/i2p/router/client/ClientConnectionRunner.java b/router/java/src/net/i2p/router/client/ClientConnectionRunner.java index 42eef3480..1c4fb2fd5 100644 --- a/router/java/src/net/i2p/router/client/ClientConnectionRunner.java +++ b/router/java/src/net/i2p/router/client/ClientConnectionRunner.java @@ -361,18 +361,21 @@ public class ClientConnectionRunner { // TunnelPool.locked_buildNewLeaseSet() ensures that leases are sorted, // so the comparison will always work. int leases = set.getLeaseCount(); - if (_currentLeaseSet != null && _currentLeaseSet.getLeaseCount() == leases) { - for (int i = 0; i < leases; i++) { - if (! _currentLeaseSet.getLease(i).getTunnelId().equals(set.getLease(i).getTunnelId())) - break; - if (! _currentLeaseSet.getLease(i).getGateway().equals(set.getLease(i).getGateway())) - break; - if (i == leases - 1) { - if (_log.shouldLog(Log.INFO)) - _log.info("Requested leaseSet hasn't changed"); - if (onCreateJob != null) - _context.jobQueue().addJob(onCreateJob); - return; // no change + // synch so _currentLeaseSet isn't changed out from under us + synchronized (this) { + if (_currentLeaseSet != null && _currentLeaseSet.getLeaseCount() == leases) { + for (int i = 0; i < leases; i++) { + if (! _currentLeaseSet.getLease(i).getTunnelId().equals(set.getLease(i).getTunnelId())) + break; + if (! _currentLeaseSet.getLease(i).getGateway().equals(set.getLease(i).getGateway())) + break; + if (i == leases - 1) { + if (_log.shouldLog(Log.INFO)) + _log.info("Requested leaseSet hasn't changed"); + if (onCreateJob != null) + _context.jobQueue().addJob(onCreateJob); + return; // no change + } } } } From 07aa07981dee8e81ecdeee6bdad8c420f8698e83 Mon Sep 17 00:00:00 2001 From: zzz Date: Sun, 31 Oct 2010 14:52:09 +0000 Subject: [PATCH 26/31] * logs.jsp: Add message if wrapper log not found (ticket #103) --- apps/routerconsole/java/src/net/i2p/router/web/LogsHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/routerconsole/java/src/net/i2p/router/web/LogsHelper.java b/apps/routerconsole/java/src/net/i2p/router/web/LogsHelper.java index 9955ffeed..f010be834 100644 --- a/apps/routerconsole/java/src/net/i2p/router/web/LogsHelper.java +++ b/apps/routerconsole/java/src/net/i2p/router/web/LogsHelper.java @@ -31,7 +31,7 @@ public class LogsHelper extends HelperBase { } String str = FileUtil.readTextFile(f.getAbsolutePath(), 250, false); if (str == null) - return ""; + return _("File not found") + ": " + f.getAbsolutePath() + ""; else { str = str.replace("&", "&").replace("<", "<").replace(">", ">"); return _("File location") + ": " + f.getAbsolutePath() + " 
    " + str + "
    "; From ba4f6608e4954d224dc4f3384bc64e63c7458f78 Mon Sep 17 00:00:00 2001 From: zzz Date: Sun, 31 Oct 2010 14:52:21 +0000 Subject: [PATCH 27/31] update nbsp comment --- .../java/src/net/i2p/router/web/ConfigLoggingHelper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/routerconsole/java/src/net/i2p/router/web/ConfigLoggingHelper.java b/apps/routerconsole/java/src/net/i2p/router/web/ConfigLoggingHelper.java index f316e7dc6..7bd4e4546 100644 --- a/apps/routerconsole/java/src/net/i2p/router/web/ConfigLoggingHelper.java +++ b/apps/routerconsole/java/src/net/i2p/router/web/ConfigLoggingHelper.java @@ -25,7 +25,8 @@ public class ConfigLoggingHelper extends HelperBase { int bytes = _context.logManager().getFileSize(); if (bytes <= 0) return "1.00 MB"; // " " comes back in the POST as 0xc2 0xa0 - // which is not recognized as whitespace and who knows why + // non-breaking space is U+00A0 which is 0xc2 0xa0 in UTF-8. + // we could figure out where the UTF-8 problem is but why bother. return DataHelper.formatSize2(bytes).replace(" ", " ") + 'B'; } public String getLogLevelTable() { From 2f54ec61bdd7bb55f014b46e5c483d5af52ad0cf Mon Sep 17 00:00:00 2001 From: zzz Date: Mon, 1 Nov 2010 14:30:24 +0000 Subject: [PATCH 28/31] logging tweak --- .../src/net/i2p/router/transport/ntcp/NTCPConnection.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/router/java/src/net/i2p/router/transport/ntcp/NTCPConnection.java b/router/java/src/net/i2p/router/transport/ntcp/NTCPConnection.java index 52c0218fc..58ebe3634 100644 --- a/router/java/src/net/i2p/router/transport/ntcp/NTCPConnection.java +++ b/router/java/src/net/i2p/router/transport/ntcp/NTCPConnection.java @@ -615,8 +615,8 @@ public class NTCPConnection implements FIFOBandwidthLimiter.CompleteListener { if (_log.shouldLog(Log.INFO)) _log.info("Type " + msg.getMessage().getType() + " pri " + msg.getPriority() + " slot " + slot); boolean removed = _outbound.remove(msg); - if ((!removed) && _log.shouldLog(Log.ERROR)) - _log.info("Already removed??? " + msg.getMessage().getType()); + if ((!removed) && _log.shouldLog(Log.WARN)) + _log.warn("Already removed??? " + msg.getMessage().getType()); } _currentOutbound = msg; } From 8d13bcbac08a49ba26f3f5d5f6b617d488665030 Mon Sep 17 00:00:00 2001 From: zzz Date: Mon, 1 Nov 2010 14:30:57 +0000 Subject: [PATCH 29/31] discourage b32 --- apps/i2ptunnel/jsp/editClient.jsp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/i2ptunnel/jsp/editClient.jsp b/apps/i2ptunnel/jsp/editClient.jsp index 2e6d09e43..485c6758c 100644 --- a/apps/i2ptunnel/jsp/editClient.jsp +++ b/apps/i2ptunnel/jsp/editClient.jsp @@ -159,7 +159,7 @@ %> - (<%=intl._("name or destination")%>) + (<%=intl._("name or destination")%>; <%=intl._("b32 not recommended")%>) <% } %> <% if (!"streamrclient".equals(tunnelType)) { %> From 3678aa157e1e2c288ad42a30691f4fee7d538a43 Mon Sep 17 00:00:00 2001 From: zzz Date: Mon, 1 Nov 2010 14:35:01 +0000 Subject: [PATCH 30/31] * i2psnark: - Limit number of parallel requests of a single piece when in the end game - Shorten and weight the speed tracker so the display is more reflective of current speed --- .../java/src/org/klomp/snark/Peer.java | 47 ++++++------------ .../src/org/klomp/snark/PeerCoordinator.java | 48 ++++++++++++++----- .../java/src/org/klomp/snark/PeerState.java | 12 +++++ history.txt | 12 +++++ .../src/net/i2p/router/RouterVersion.java | 2 +- 5 files changed, 77 insertions(+), 44 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/Peer.java b/apps/i2psnark/java/src/org/klomp/snark/Peer.java index 0445ab205..55b6dcb2c 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Peer.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Peer.java @@ -56,8 +56,8 @@ public class Peer implements Comparable private long _id; final static long CHECK_PERIOD = PeerCoordinator.CHECK_PERIOD; // 40 seconds final static int RATE_DEPTH = PeerCoordinator.RATE_DEPTH; // make following arrays RATE_DEPTH long - private long uploaded_old[] = {-1,-1,-1,-1,-1,-1}; - private long downloaded_old[] = {-1,-1,-1,-1,-1,-1}; + private long uploaded_old[] = {-1,-1,-1}; + private long downloaded_old[] = {-1,-1,-1}; /** * Creates a disconnected peer given a PeerID, your own id and the @@ -403,6 +403,15 @@ public class Peer implements Comparable s.cancelPiece(piece); } + /** + * Are we currently requesting the piece? + * @since 0.8.1 + */ + boolean isRequesting(int p) { + PeerState s = state; + return s != null && s.isRequesting(p); + } + /** * Update the request queue. * Call after adding wanted pieces. @@ -572,17 +581,8 @@ public class Peer implements Comparable */ public void setRateHistory(long up, long down) { - setRate(up, uploaded_old); - setRate(down, downloaded_old); - } - - private void setRate(long val, long array[]) - { - synchronized(array) { - for (int i = RATE_DEPTH-1; i > 0; i--) - array[i] = array[i-1]; - array[0] = val; - } + PeerCoordinator.setRate(up, uploaded_old); + PeerCoordinator.setRate(down, downloaded_old); } /** @@ -590,28 +590,11 @@ public class Peer implements Comparable */ public long getUploadRate() { - return getRate(uploaded_old); + return PeerCoordinator.getRate(uploaded_old); } public long getDownloadRate() { - return getRate(downloaded_old); + return PeerCoordinator.getRate(downloaded_old); } - - private long getRate(long array[]) - { - long rate = 0; - int i = 0; - synchronized(array) { - for ( ; i < RATE_DEPTH; i++){ - if (array[i] < 0) - break; - rate += array[i]; - } - } - if (i == 0) - return 0; - return rate / (i * CHECK_PERIOD / 1000); - } - } diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java index 0fa74e954..510272459 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java @@ -57,9 +57,9 @@ public class PeerCoordinator implements PeerListener private long uploaded; private long downloaded; - final static int RATE_DEPTH = 6; // make following arrays RATE_DEPTH long - private long uploaded_old[] = {-1,-1,-1,-1,-1,-1}; - private long downloaded_old[] = {-1,-1,-1,-1,-1,-1}; + final static int RATE_DEPTH = 3; // make following arrays RATE_DEPTH long + private long uploaded_old[] = {-1,-1,-1}; + private long downloaded_old[] = {-1,-1,-1}; // synchronize on this when changing peers or downloaders final List peers = new ArrayList(); @@ -193,7 +193,7 @@ public class PeerCoordinator implements PeerListener setRate(down, downloaded_old); } - private static void setRate(long val, long array[]) + static void setRate(long val, long array[]) { synchronized(array) { for (int i = RATE_DEPTH-1; i > 0; i--) @@ -224,20 +224,23 @@ public class PeerCoordinator implements PeerListener return (r * 1000) / CHECK_PERIOD; } - private long getRate(long array[]) + static long getRate(long array[]) { long rate = 0; int i = 0; + int factor = 0; synchronized(array) { for ( ; i < RATE_DEPTH; i++) { if (array[i] < 0) break; - rate += array[i]; + int f = RATE_DEPTH - i; + rate += array[i] * f; + factor += f; } } if (i == 0) return 0; - return rate / (i * CHECK_PERIOD / 1000); + return rate / (factor * i * CHECK_PERIOD / 1000); } public MetaInfo getMetaInfo() @@ -509,6 +512,12 @@ public class PeerCoordinator implements PeerListener */ private static final int END_GAME_THRESHOLD = 8; + /** + * Max number of peers to get a piece from when in end game + * @since 0.8.1 + */ + private static final int MAX_PARALLEL_REQUESTS = 4; + /** * Returns one of pieces in the given BitField that is still wanted or * -1 if none of the given pieces are wanted. @@ -551,15 +560,32 @@ public class PeerCoordinator implements PeerListener if (wantedPieces.size() > END_GAME_THRESHOLD) return -1; // nothing to request and not in end game // let's not all get on the same piece + // Even better would be to sort by number of requests Collections.shuffle(requested, _random); Iterator it2 = requested.iterator(); while (piece == null && it2.hasNext()) { Piece p = it2.next(); - if (havePieces.get(p.getId())) - { + if (havePieces.get(p.getId())) { + // limit number of parallel requests + int requestedCount = 0; + synchronized(peers) { + for (Peer pr : peers) { + if (pr.isRequesting(p.getId())) { + if (pr.equals(peer)) { + // don't give it to him again + requestedCount = MAX_PARALLEL_REQUESTS; + break; + } + if (++requestedCount >= MAX_PARALLEL_REQUESTS) + break; + } + } + } + if (requestedCount >= MAX_PARALLEL_REQUESTS) + continue; piece = p; - } + } } if (piece == null) { if (_log.shouldLog(Log.WARN)) @@ -568,7 +594,7 @@ public class PeerCoordinator implements PeerListener // + " wanted = " + wantedPieces + " peerHas = " + havePieces); return -1; //If we still can't find a piece we want, so be it. } else { - // Should be a lot smarter here - limit # of parallel attempts and + // Should be a lot smarter here - // share blocks rather than starting from 0 with each peer. // This is where the flaws of the snark data model are really exposed. // Could also randomize within the duplicate set rather than strict rarest-first diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerState.java b/apps/i2psnark/java/src/org/klomp/snark/PeerState.java index cadcbf866..4d34736a8 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerState.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerState.java @@ -470,6 +470,18 @@ class PeerState } } + /** + * Are we currently requesting the piece? + * @since 0.8.1 + */ + synchronized boolean isRequesting(int piece) { + for (Request req : outstandingRequests) { + if (req.piece == piece) + return true; + } + return false; + } + /** * Starts or resumes requesting pieces. * @param resend should we resend outstanding requests? diff --git a/history.txt b/history.txt index ab77d569f..fcd33d6b4 100644 --- a/history.txt +++ b/history.txt @@ -1,3 +1,15 @@ +2010-11-01 zzz + * ClientConnectionRunner: Add synch to fix race causing AIOOBE + (http://forum.i2p/viewtopic.php?t=5061) + * configlogging.jsp: Parse log limit with current locale + (ticket #118) + * i2psnark: + - Limit number of parallel requests of a single piece when in the end game + - Shorten and weight the speed tracker so the display is more + reflective of current speed + * logs.jsp: Add message if wrapper log not found + (ticket #103) + 2010-10-30 zzz * i2psnark: - Priority mapping bugfix diff --git a/router/java/src/net/i2p/router/RouterVersion.java b/router/java/src/net/i2p/router/RouterVersion.java index c10128fe6..fecba78d6 100644 --- a/router/java/src/net/i2p/router/RouterVersion.java +++ b/router/java/src/net/i2p/router/RouterVersion.java @@ -18,7 +18,7 @@ public class RouterVersion { /** deprecated */ public final static String ID = "Monotone"; public final static String VERSION = CoreVersion.VERSION; - public final static long BUILD = 6; + public final static long BUILD = 7; /** for example "-test" */ public final static String EXTRA = ""; From ddc86b54c7e14f5d386cf56c9da3c5a92a7a8d18 Mon Sep 17 00:00:00 2001 From: zzz Date: Mon, 1 Nov 2010 22:04:10 +0000 Subject: [PATCH 31/31] fix snark rates 3x too low --- apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java | 2 +- router/java/src/net/i2p/router/RouterVersion.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java index 510272459..c8f8d7d7f 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java @@ -240,7 +240,7 @@ public class PeerCoordinator implements PeerListener } if (i == 0) return 0; - return rate / (factor * i * CHECK_PERIOD / 1000); + return rate / (factor * CHECK_PERIOD / 1000); } public MetaInfo getMetaInfo() diff --git a/router/java/src/net/i2p/router/RouterVersion.java b/router/java/src/net/i2p/router/RouterVersion.java index fecba78d6..a6204817e 100644 --- a/router/java/src/net/i2p/router/RouterVersion.java +++ b/router/java/src/net/i2p/router/RouterVersion.java @@ -18,7 +18,7 @@ public class RouterVersion { /** deprecated */ public final static String ID = "Monotone"; public final static String VERSION = CoreVersion.VERSION; - public final static long BUILD = 7; + public final static long BUILD = 8; /** for example "-test" */ public final static String EXTRA = "";