diff --git a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java index befa2acb4..2731753cc 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java +++ b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java @@ -105,6 +105,7 @@ public class SnarkManager implements CompleteListener, ClientApp { private static final String PROP_META_UPLOADED = "uploaded"; private static final String PROP_META_ADDED = "added"; private static final String PROP_META_COMPLETED = "completed"; + private static final String PROP_META_INORDER = "inOrder"; private static final String PROP_META_MAGNET = "magnet"; private static final String PROP_META_MAGNET_DN = "magnet_dn"; private static final String PROP_META_MAGNET_TR = "magnet_tr"; @@ -1759,7 +1760,7 @@ public class SnarkManager implements CompleteListener, ClientApp { addMessage(_t("Torrent with this info hash is already running: {0}", snark.getBaseName())); return false; } else if (bitfield != null) { - saveTorrentStatus(metainfo, bitfield, null, baseFile, true, 0, true); // no file priorities + saveTorrentStatus(metainfo, bitfield, null, false, baseFile, true, 0, true); // no file priorities } // so addTorrent won't recheck if (filename == null) { @@ -1898,19 +1899,21 @@ public class SnarkManager implements CompleteListener, ClientApp { return; Properties config = getConfig(snark); String pri = config.getProperty(PROP_META_PRIORITY); - if (pri == null) - return; - int filecount = metainfo.getFiles().size(); - int[] rv = new int[filecount]; - String[] arr = DataHelper.split(pri, ","); - for (int i = 0; i < filecount && i < arr.length; i++) { - if (arr[i].length() > 0) { - try { - rv[i] = Integer.parseInt(arr[i]); - } catch (Throwable t) {} + if (pri != null) { + int filecount = metainfo.getFiles().size(); + int[] rv = new int[filecount]; + String[] arr = DataHelper.split(pri, ","); + for (int i = 0; i < filecount && i < arr.length; i++) { + if (arr[i].length() > 0) { + try { + rv[i] = Integer.parseInt(arr[i]); + } catch (Throwable t) {} + } } + storage.setFilePriorities(rv); } - storage.setFilePriorities(rv); + boolean inOrder = Boolean.parseBoolean(config.getProperty(PROP_META_INORDER)); + storage.setInOrder(inOrder); } /** @@ -2017,7 +2020,7 @@ public class SnarkManager implements CompleteListener, ClientApp { Storage storage = snark.getStorage(); if (meta == null || storage == null) return; - saveTorrentStatus(meta, storage.getBitField(), storage.getFilePriorities(), + saveTorrentStatus(meta, storage.getBitField(), storage.getFilePriorities(), storage.getInOrder(), storage.getBase(), storage.getPreserveFileNames(), snark.getUploaded(), snark.isStopped(), comments); } @@ -2034,24 +2037,24 @@ public class SnarkManager implements CompleteListener, ClientApp { * @param priorities may be null * @param base may be null */ - private void saveTorrentStatus(MetaInfo metainfo, BitField bitfield, int[] priorities, + private void saveTorrentStatus(MetaInfo metainfo, BitField bitfield, int[] priorities, boolean inOrder, File base, boolean preserveNames, long uploaded, boolean stopped) { - saveTorrentStatus(metainfo, bitfield, priorities, base, preserveNames, uploaded, stopped, null); + saveTorrentStatus(metainfo, bitfield, priorities, inOrder, base, preserveNames, uploaded, stopped, null); } /* * @param comments null for no change * @since 0.9.31 */ - private void saveTorrentStatus(MetaInfo metainfo, BitField bitfield, int[] priorities, + private void saveTorrentStatus(MetaInfo metainfo, BitField bitfield, int[] priorities, boolean inOrder, File base, boolean preserveNames, long uploaded, boolean stopped, Boolean comments) { synchronized (_configLock) { - locked_saveTorrentStatus(metainfo, bitfield, priorities, base, preserveNames, uploaded, stopped, comments); + locked_saveTorrentStatus(metainfo, bitfield, priorities, inOrder, base, preserveNames, uploaded, stopped, comments); } } - private void locked_saveTorrentStatus(MetaInfo metainfo, BitField bitfield, int[] priorities, + private void locked_saveTorrentStatus(MetaInfo metainfo, BitField bitfield, int[] priorities, boolean inOrder, File base, boolean preserveNames, long uploaded, boolean stopped, Boolean comments) { byte[] ih = metainfo.getInfoHash(); @@ -2077,6 +2080,7 @@ public class SnarkManager implements CompleteListener, ClientApp { config.setProperty(PROP_META_UPLOADED, Long.toString(uploaded)); boolean running = !stopped; config.setProperty(PROP_META_RUNNING, Boolean.toString(running)); + config.setProperty(PROP_META_INORDER, Boolean.toString(inOrder)); if (base != null) config.setProperty(PROP_META_BASE, base.getAbsolutePath()); if (comments != null) @@ -2095,7 +2099,9 @@ public class SnarkManager implements CompleteListener, ClientApp { // generate string like -5,,4,3,,,,,,-2 where no number is zero. StringBuilder buf = new StringBuilder(2 * priorities.length); for (int i = 0; i < priorities.length; i++) { - if (priorities[i] != 0) + // only output if !inOrder || !skipped so the string isn't too long + if (priorities[i] != 0 && + (!inOrder || priorities[i] < 0)) buf.append(Integer.toString(priorities[i])); if (i != priorities.length - 1) buf.append(','); @@ -2443,7 +2449,7 @@ public class SnarkManager implements CompleteListener, ClientApp { MetaInfo meta = snark.getMetaInfo(); Storage storage = snark.getStorage(); if (meta != null && storage != null) - saveTorrentStatus(meta, storage.getBitField(), storage.getFilePriorities(), + saveTorrentStatus(meta, storage.getBitField(), storage.getFilePriorities(), storage.getInOrder(), storage.getBase(), storage.getPreserveFileNames(), snark.getUploaded(), snark.isStopped()); } @@ -2467,7 +2473,7 @@ public class SnarkManager implements CompleteListener, ClientApp { snark.stopTorrent(); return null; } - saveTorrentStatus(meta, storage.getBitField(), null, + saveTorrentStatus(meta, storage.getBitField(), null, false, storage.getBase(), storage.getPreserveFileNames(), 0, snark.isStopped()); // temp for addMessage() in case canonical throws diff --git a/apps/i2psnark/java/src/org/klomp/snark/Storage.java b/apps/i2psnark/java/src/org/klomp/snark/Storage.java index ab181172a..902ae8a37 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Storage.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Storage.java @@ -25,11 +25,14 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.RandomAccessFile; +import java.io.Serializable; import java.nio.charset.Charset; import java.nio.charset.CharsetEncoder; import java.security.MessageDigest; +import java.text.Collator; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -72,6 +75,7 @@ public class Storage implements Closeable private final boolean _preserveFileNames; private boolean changed; private volatile boolean _isChecking; + private boolean _inOrder; private final AtomicInteger _allocateCount = new AtomicInteger(); private final AtomicInteger _checkProgress = new AtomicInteger(); @@ -82,6 +86,8 @@ public class Storage implements Closeable /** The maximum number of pieces in a torrent. */ public static final int MAX_PIECES = 32*1024; public static final long MAX_TOTAL_SIZE = MAX_PIECE_SIZE * (long) MAX_PIECES; + public static final int PRIORITY_SKIP = -9; + public static final int PRIORITY_NORMAL = 0; private static final Map _filterNameCache = new ConcurrentHashMap(); @@ -145,7 +151,7 @@ public class Storage implements Closeable _torrentFiles = getFiles(baseFile); long total = 0; - ArrayList lengthsList = new ArrayList(); + ArrayList lengthsList = new ArrayList(_torrentFiles.size()); for (TorrentFile tf : _torrentFiles) { long length = tf.length; @@ -178,7 +184,7 @@ public class Storage implements Closeable bitfield = new BitField(pieces); needed = 0; - List> files = new ArrayList>(); + List> files = new ArrayList>(_torrentFiles.size()); for (TorrentFile tf : _torrentFiles) { List file = new ArrayList(); @@ -494,6 +500,60 @@ public class Storage implements Closeable } } + /** + * @return as last set, default false + * @since 0.9.36 + */ + public boolean getInOrder() { + return _inOrder; + } + + /** + * Call AFTER setFilePriorites() so we know what's skipped + * @param yes enable or not + * @since 0.9.36 + */ + public void setInOrder(boolean yes) { + if (yes == _inOrder) + return; + _inOrder = yes; + if (complete() || metainfo.getFiles() == null) + return; + if (yes) { + List sorted = _torrentFiles; + int sz = sorted.size(); + if (sz > 1) { + sorted = new ArrayList(sorted); + Collections.sort(sorted, new FileNameComparator()); + } + for (int i = 0; i < sz; i++) { + TorrentFile tf = sorted.get(i); + // higher number is higher priority + if (tf.priority >= PRIORITY_NORMAL) + tf.priority = sz - i; + } + } else { + for (TorrentFile tf : _torrentFiles) { + if (tf.priority > PRIORITY_NORMAL) + tf.priority = PRIORITY_NORMAL; + } + } + } + + /** + * Sort with locale comparator. + * (not using TorrentFile.compareTo()) + * @since 0.9.36 + */ + private static class FileNameComparator implements Comparator, Serializable { + + private final Collator c = Collator.getInstance(); + + public int compare(TorrentFile l, TorrentFile r) { + return c.compare(l.toString(), r.toString()); + } + } + /** * Call setPriority() for all changed files first, * then call this. @@ -523,6 +583,25 @@ public class Storage implements Closeable } rv[i] = pri; } + if (_inOrder) { + // Do a second pass to set the priority of the pieces within each file + // this only works because MAX_PIECES * MAX_FILES_PER_TORRENT < Integer.MAX_VALUE + // the base file priority + int pri = PRIORITY_SKIP; + for (int i = 0; i < rv.length; i++) { + int val = rv[i]; + if (val <= PRIORITY_NORMAL) + continue; + if (val != pri) { + pri = val; + // new file + rv[i] *= MAX_PIECES; + } else { + // same file, decrement priority from previous piece + rv[i] = rv[i-1] - 1; + } + } + } return rv; } @@ -544,7 +623,7 @@ public class Storage implements Closeable long rv = 0; final int end = pri.length - 1; for (int i = 0; i <= end; i++) { - if (pri[i] <= -9 && !bitfield.get(i)) { + if (pri[i] <= PRIORITY_SKIP && !bitfield.get(i)) { rv += (i != end) ? piece_size : metainfo.getPieceLength(i); } } diff --git a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java index 9a4460201..45fa70f47 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java +++ b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java @@ -2931,7 +2931,8 @@ public class I2PSnarkServlet extends BasicServlet { if (val != null) { String nonce = val[0]; if (String.valueOf(_nonce).equals(nonce)) { - if (postParams.get("savepri") != null) { + if (postParams.get("savepri") != null || + postParams.get("setInOrderEnabled") != null) { savePriorities(snark, postParams); } else if (postParams.get("addComment") != null) { saveComments(snark, postParams); @@ -2974,7 +2975,8 @@ public class I2PSnarkServlet extends BasicServlet { } boolean showStopStart = snark != null; - boolean showPriority = snark != null && snark.getStorage() != null && !snark.getStorage().complete() && + Storage storage = snark != null ? snark.getStorage() : null; + boolean showPriority = storage != null && !storage.complete() && r.isDirectory(); StringBuilder buf=new StringBuilder(4096); @@ -3036,13 +3038,13 @@ public class I2PSnarkServlet extends BasicServlet { .append(": ") .append(DataHelper.escapeHTML(fullPath)) .append("\n"); - if (snark.getStorage() != null) { + if (storage != null) { buf.append(""); toThemeImg(buf, "file"); buf.append("") .append(_t("Data location")) .append(": ") - .append(DataHelper.escapeHTML(snark.getStorage().getBase().getPath())) + .append(DataHelper.escapeHTML(storage.getBase().getPath())) .append("\n"); } String hex = I2PSnarkUtil.toHex(snark.getInfoHash()); @@ -3285,6 +3287,23 @@ public class I2PSnarkServlet extends BasicServlet { } buf.append("\n"); } + + boolean showInOrder = storage != null && !storage.complete() && + meta != null && meta.getFiles() != null && meta.getFiles().size() > 1; + if (showInOrder) { + buf.append(""); + buf.append(_t("Download files in order")); + buf.append(":" + + "" + + "\n"); + } } else { // snark == null // shouldn't happen @@ -3323,7 +3342,6 @@ public class I2PSnarkServlet extends BasicServlet { return buf.toString(); } - Storage storage = snark != null ? snark.getStorage() : null; List fileList = new ArrayList(ls.length); // precompute remaining for all files for efficiency long[] remainingArray = (storage != null) ? storage.remaining() : null; @@ -3424,6 +3442,7 @@ public class I2PSnarkServlet extends BasicServlet { // DateFormat.MEDIUM); boolean showSaveButton = false; boolean rowEven = true; + boolean inOrder = storage != null && storage.getInOrder(); for (Sorters.FileAndIndex fai : fileList) { //String encoded = encodePath(ls[i].getName()); @@ -3447,7 +3466,7 @@ public class I2PSnarkServlet extends BasicServlet { complete = true; //status = toImg("tick") + ' ' + _t("Directory"); } else { - if (snark == null || snark.getStorage() == null) { + if (storage == null) { // Assume complete, perhaps he removed a completed torrent but kept a bookmark complete = true; status = toImg("cancel") + ' ' + _t("Torrent not found?"); @@ -3527,16 +3546,18 @@ public class I2PSnarkServlet extends BasicServlet { if (showPriority) { buf.append(""); if ((!complete) && (!item.isDirectory())) { - buf.append(""); + if (!inOrder) { + buf.append(""); + } buf.append(""); @@ -3555,9 +3576,12 @@ public class I2PSnarkServlet extends BasicServlet { } if (showSaveButton) { buf.append("" + - "") - .append(toImg("clock_red")).append(_t("Set all high")).append("\n" + - "") + ""); + if (!inOrder) { + buf.append("") + .append(toImg("clock_red")).append(_t("Set all high")).append("\n"); + } + buf.append("") .append(toImg("clock")).append(_t("Set all normal")).append("\n" + "") .append(toImg("cancel")).append(_t("Skip all")).append("\n" + @@ -3926,6 +3950,8 @@ public class I2PSnarkServlet extends BasicServlet { } catch (Throwable t) { t.printStackTrace(); } } } + if (postParams.get("setInOrderEnabled") != null) + storage.setInOrder(postParams.get("enableInOrder") != null); snark.updatePiecePriorities(); _manager.saveTorrentStatus(snark); } diff --git a/apps/streaming/java/src/net/i2p/client/streaming/impl/ConnectionManager.java b/apps/streaming/java/src/net/i2p/client/streaming/impl/ConnectionManager.java index c57992aaa..42cece4f5 100644 --- a/apps/streaming/java/src/net/i2p/client/streaming/impl/ConnectionManager.java +++ b/apps/streaming/java/src/net/i2p/client/streaming/impl/ConnectionManager.java @@ -97,11 +97,10 @@ class ConnectionManager { _connectionHandler = new ConnectionHandler(_context, this, _timer); _tcbShare = new TCBShare(_context, _timer); // PROTO_ANY is for backward compatibility (pre-0.7.1) - // TODO change proto to PROTO_STREAMING someday. - // Right now we get everything, and rely on Datagram to specify PROTO_UDP. // PacketQueue has sent PROTO_STREAMING since the beginning of mux support (0.7.1) // As of 0.9.1, new option to enforce streaming protocol, off by default // As of 0.9.1, listen on configured port (default 0 = all) + // enforce protocol default changed to true in 0.9.36 int protocol = defaultOptions.getEnforceProtocol() ? I2PSession.PROTO_STREAMING : I2PSession.PROTO_ANY; _session.addMuxedSessionListener(_messageHandler, protocol, defaultOptions.getLocalPort()); _outboundQueue = new PacketQueue(_context, _timer); diff --git a/apps/streaming/java/src/net/i2p/client/streaming/impl/ConnectionOptions.java b/apps/streaming/java/src/net/i2p/client/streaming/impl/ConnectionOptions.java index dbf12fb43..6f88b5bc7 100644 --- a/apps/streaming/java/src/net/i2p/client/streaming/impl/ConnectionOptions.java +++ b/apps/streaming/java/src/net/i2p/client/streaming/impl/ConnectionOptions.java @@ -155,9 +155,10 @@ class ConnectionOptions extends I2PSocketOptionsImpl { /** * If PROTO is enforced, we cannot communicate with destinations earlier than version 0.7.1. + * Default true as of 0.9.36. * @since 0.9.1 */ - private static final boolean DEFAULT_ENFORCE_PROTO = false; + private static final boolean DEFAULT_ENFORCE_PROTO = true; private final int _trend[] = new int[TREND_COUNT]; @@ -523,7 +524,7 @@ class ConnectionOptions extends I2PSocketOptionsImpl { /** * Do we receive all traffic, or only traffic marked with I2PSession.PROTO_STREAMING (6) ? - * Default false. + * Default true. * If PROTO is enforced, we cannot communicate with destinations earlier than version 0.7.1 * (released March 2009), which is when streaming started sending the PROTO_STREAMING indication. * Set to true if you are running multiple protocols on a single Destination. diff --git a/build.properties b/build.properties index c564a7f10..b1326e5f7 100644 --- a/build.properties +++ b/build.properties @@ -30,10 +30,12 @@ sloccount.report.file=sloccount.sc # Building EXEs in x64 Linux requires that 32bit libraries are installed. In Debian, # for example, installing the libc6-i386 package will satisfy this requirement. - # Uncomment the next line to prevent building EXEs (changing it to false will have no impact) #noExe=true +# IzPack 5.1.x install dir +#izpack5.home=/PATH/TO/IzPack + # Change this to false if you don't have gettext or you want to prevent it from running during the build require.gettext=true diff --git a/build.xml b/build.xml index 9973d6fde..c88bdd4b4 100644 --- a/build.xml +++ b/build.xml @@ -24,6 +24,7 @@ + @@ -79,7 +80,7 @@ - + @@ -1762,18 +1763,36 @@ + - + + + + + + + + + + + + + + + + + + + + + + + + @@ -1786,22 +1805,36 @@ + + + + + + + + + + + + + + @@ -1845,6 +1878,7 @@ + @@ -1853,6 +1887,15 @@ + + + + + + + + + diff --git a/core/java/src/freenet/support/CPUInformation/AMDInfoImpl.java b/core/java/src/freenet/support/CPUInformation/AMDInfoImpl.java index da71628ee..d5c7f5886 100644 --- a/core/java/src/freenet/support/CPUInformation/AMDInfoImpl.java +++ b/core/java/src/freenet/support/CPUInformation/AMDInfoImpl.java @@ -477,6 +477,23 @@ class AMDInfoImpl extends CPUIDCPUInfo implements AMDCPUInfo modelString = "Ryzen model " + model; } break; + + // http://lkml.iu.edu/hypermail/linux/kernel/1806.1/00730.html + // untested + case 24: { + isK6Compatible = true; + isK6_2_Compatible = true; + isK6_3_Compatible = true; + isAthlonCompatible = true; + isAthlon64Compatible = true; + // Pending testing of the bulldozer jbigi + //isPiledriverCompatible = true; + //isSteamrollerCompatible = true; + //isExcavatorCompatible = true; + //isBulldozerCompatible = true; + modelString = "Hygon Dhyana model " + model; + } + break; } return modelString; } diff --git a/core/java/src/freenet/support/CPUInformation/CPUID.java b/core/java/src/freenet/support/CPUInformation/CPUID.java index ca8c092cd..82df6138f 100644 --- a/core/java/src/freenet/support/CPUInformation/CPUID.java +++ b/core/java/src/freenet/support/CPUInformation/CPUID.java @@ -310,7 +310,8 @@ public class CPUID { return new VIAInfoImpl(); if(!isX86) throw new UnknownCPUException("Failed to read CPU information from the system. The CPUID instruction exists on x86 CPUs only."); - if(id.equals("AuthenticAMD")) + // http://lkml.iu.edu/hypermail/linux/kernel/1806.1/00730.html + if(id.equals("AuthenticAMD") || id.equals("HygonGenuine")) return new AMDInfoImpl(); if(id.equals("GenuineIntel")) return new IntelInfoImpl(); diff --git a/core/java/src/net/i2p/util/ByteCache.java b/core/java/src/net/i2p/util/ByteCache.java index 3600994b9..dbd0cf315 100644 --- a/core/java/src/net/i2p/util/ByteCache.java +++ b/core/java/src/net/i2p/util/ByteCache.java @@ -2,9 +2,7 @@ package net.i2p.util; import java.util.Arrays; import java.util.Map; -import java.util.Queue; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.LinkedBlockingQueue; import net.i2p.I2PAppContext; import net.i2p.data.ByteArray; @@ -22,13 +20,14 @@ import net.i2p.data.ByteArray; Size Max MaxMem From 1K 32 32K tunnel TrivialPreprocessor + *changed to 512 since we disabled resize() 1K 512 512K tunnel FragmentHandler 1K 512 512K I2NP TunnelDataMessage 1K 512 512K tunnel FragmentedMessage - 1730 128 216K streaming MessageOutputStream + 1572 64 100K UDP InboundMessageState - 2K 64 128K UDP IMS + 1730 128 216K streaming MessageOutputStream 4K 32 128K I2PTunnelRunner @@ -49,7 +48,7 @@ import net.i2p.data.ByteArray; * * */ -public final class ByteCache { +public final class ByteCache extends TryCache { //private static final Log _log = I2PAppContext.getGlobalContext().logManager().getLog(ByteCache.class); private static final Map _caches = new ConcurrentHashMap(16); @@ -83,10 +82,13 @@ public final class ByteCache { if (cacheSize * size > MAX_CACHE) cacheSize = MAX_CACHE / size; Integer sz = Integer.valueOf(size); - ByteCache cache = _caches.get(sz); - if (cache == null) { - cache = new ByteCache(cacheSize, size); - _caches.put(sz, cache); + ByteCache cache; + synchronized(_caches) { + cache = _caches.get(sz); + if (cache == null) { + cache = new ByteCache(cacheSize, size); + _caches.put(sz, cache); + } } cache.resize(cacheSize); //I2PAppContext.getGlobalContext().logManager().getLog(ByteCache.class).error("ByteCache size: " + size + " max: " + cacheSize, new Exception("from")); @@ -103,109 +105,85 @@ public final class ByteCache { //_log.warn("WARNING: Low memory, clearing byte caches"); } - /** list of available and available entries */ - private volatile Queue _available; - private int _maxCached; private final int _entrySize; - private volatile long _lastOverflow; - - /** do we actually want to cache? Warning - setting to false may NPE, this should be fixed or removed */ - private static final boolean _cache = true; /** how often do we cleanup the cache */ private static final int CLEANUP_FREQUENCY = 33*1000; /** if we haven't exceeded the cache size in 2 minutes, cut our cache in half */ private static final long EXPIRE_PERIOD = 2*60*1000; + /** @since 0.9.36 */ + private static class ByteArrayFactory implements TryCache.ObjectFactory { + private final int sz; + + ByteArrayFactory(int entrySize) { + sz = entrySize; + } + + public ByteArray newInstance() { + byte data[] = new byte[sz]; + ByteArray rv = new ByteArray(data); + rv.setValid(0); + return rv; + } + } + private ByteCache(int maxCachedEntries, int entrySize) { - if (_cache) - _available = new LinkedBlockingQueue(maxCachedEntries); - _maxCached = maxCachedEntries; + super(new ByteArrayFactory(entrySize), maxCachedEntries); _entrySize = entrySize; - _lastOverflow = -1; - SimpleTimer2.getInstance().addPeriodicEvent(new Cleanup(), CLEANUP_FREQUENCY + (entrySize % 777)); //stagger + int stagger = SystemVersion.isAndroid() ? 0 : (entrySize % 777); + SimpleTimer2.getInstance().addPeriodicEvent(new Cleanup(), CLEANUP_FREQUENCY + stagger); I2PAppContext.getGlobalContext().statManager().createRateStat("byteCache.memory." + entrySize, "Memory usage (B)", "Router", new long[] { 10*60*1000 }); } private void resize(int maxCachedEntries) { - if (_maxCached >= maxCachedEntries) return; - _maxCached = maxCachedEntries; - // make a bigger one, move the cached items over - Queue newLBQ = new LinkedBlockingQueue(maxCachedEntries); - ByteArray ba; - while ((ba = _available.poll()) != null) - newLBQ.offer(ba); - _available = newLBQ; - } - - /** - * Get the next available structure, either from the cache or a brand new one. - * Returned ByteArray will have valid = 0 and offset = 0. - * Returned ByteArray may or may not be zero, depends on whether - * release(ba) or release(ba, false) was called. - * Which is a problem, you should really specify shouldZero on acquire, not release. - */ - public final ByteArray acquire() { - if (_cache) { - ByteArray rv = _available.poll(); - if (rv != null) - return rv; - } - _lastOverflow = System.currentTimeMillis(); - byte data[] = new byte[_entrySize]; - ByteArray rv = new ByteArray(data); - rv.setValid(0); - //rv.setOffset(0); - return rv; + // disabled since we're now extending TryCache } /** * Put this structure back onto the available cache for reuse * */ + @Override public final void release(ByteArray entry) { release(entry, true); } public final void release(ByteArray entry, boolean shouldZero) { - if (_cache) { - if (entry == null || entry.getData() == null) - return; - if (entry.getData().length != _entrySize) { - Log log = I2PAppContext.getGlobalContext().logManager().getLog(ByteCache.class); - if (log.shouldLog(Log.WARN)) - log.warn("Bad size", new Exception("I did it")); - return; - } - entry.setValid(0); - entry.setOffset(0); - - if (shouldZero) - Arrays.fill(entry.getData(), (byte)0x0); - _available.offer(entry); + if (entry == null || entry.getData() == null) + return; + if (entry.getData().length != _entrySize) { + Log log = I2PAppContext.getGlobalContext().logManager().getLog(ByteCache.class); + if (log.shouldLog(Log.WARN)) + log.warn("Bad size", new Exception("I did it")); + return; } - } - - /** - * Clear everything (memory pressure) - * @since 0.7.14 - */ - private void clear() { - _available.clear(); + entry.setValid(0); + entry.setOffset(0); + + if (shouldZero) + Arrays.fill(entry.getData(), (byte)0x0); + super.release(entry); } private class Cleanup implements SimpleTimer.TimedEvent { public void timeReached() { - I2PAppContext.getGlobalContext().statManager().addRateData("byteCache.memory." + _entrySize, _entrySize * _available.size(), 0); - if (System.currentTimeMillis() - _lastOverflow > EXPIRE_PERIOD) { - // we haven't exceeded the cache size in a few minutes, so lets - // shrink the cache - int toRemove = _available.size() / 2; - for (int i = 0; i < toRemove; i++) - _available.poll(); - //if ( (toRemove > 0) && (_log.shouldLog(Log.DEBUG)) ) - // _log.debug("Removing " + toRemove + " cached entries of size " + _entrySize); + int origsz; + lock.lock(); + try { + origsz = items.size(); + if (origsz > 1 && System.currentTimeMillis() - _lastUnderflow > EXPIRE_PERIOD) { + // we haven't exceeded the cache size in a few minutes, so lets + // shrink the cache + int toRemove = origsz / 2; + for (int i = 0; i < toRemove; i++) { + items.remove(items.size() - 1); + } + } + } finally { + lock.unlock(); } + I2PAppContext.getGlobalContext().statManager().addRateData("byteCache.memory." + _entrySize, _entrySize * origsz); } @Override diff --git a/core/java/src/net/i2p/util/SimpleByteCache.java b/core/java/src/net/i2p/util/SimpleByteCache.java index 4b21334b0..b187e7344 100644 --- a/core/java/src/net/i2p/util/SimpleByteCache.java +++ b/core/java/src/net/i2p/util/SimpleByteCache.java @@ -1,9 +1,6 @@ package net.i2p.util; -import java.util.Queue; -import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.LinkedBlockingQueue; /** * Like ByteCache but works directly with byte arrays, not ByteArrays. @@ -19,9 +16,6 @@ public final class SimpleByteCache { private static final int DEFAULT_SIZE = 64; - /** up to this, use ABQ to minimize object churn and for performance; above this, use LBQ for two locks */ - private static final int MAX_FOR_ABQ = 64; - /** * Get a cache responsible for arrays of the given size * @@ -60,38 +54,32 @@ public final class SimpleByteCache { bc.clear(); } - /** list of available and available entries */ - private Queue _available; - private int _maxCached; + private final TryCache _available; private final int _entrySize; + /** @since 0.9.36 */ + private static class ByteArrayFactory implements TryCache.ObjectFactory { + private final int sz; + + ByteArrayFactory(int entrySize) { + sz = entrySize; + } + + public byte[] newInstance() { + return new byte[sz]; + } + } + private SimpleByteCache(int maxCachedEntries, int entrySize) { - _maxCached = maxCachedEntries; - _available = createQueue(); + _available = new TryCache(new ByteArrayFactory(entrySize), maxCachedEntries); _entrySize = entrySize; } private void resize(int maxCachedEntries) { - if (_maxCached >= maxCachedEntries) return; - _maxCached = maxCachedEntries; - // make a bigger one, move the cached items over - Queue newLBQ = createQueue(); - byte[] ba; - while ((ba = _available.poll()) != null) - newLBQ.offer(ba); - _available = newLBQ; + // _available is now final, and getInstance() is not used anywhere, + // all call sites just use static acquire() } - /** - * @return LBQ or ABQ - * @since 0.9.2 - */ - private Queue createQueue() { - if (_entrySize <= MAX_FOR_ABQ) - return new ArrayBlockingQueue(_maxCached); - return new LinkedBlockingQueue(_maxCached); - } - /** * Get the next available array, either from the cache or a brand new one */ @@ -99,14 +87,11 @@ public final class SimpleByteCache { return getInstance(size).acquire(); } - /** + /** * Get the next available array, either from the cache or a brand new one */ private byte[] acquire() { - byte[] rv = _available.poll(); - if (rv == null) - rv = new byte[_entrySize]; - return rv; + return _available.acquire(); } /** @@ -126,7 +111,7 @@ public final class SimpleByteCache { return; // should be safe without this //Arrays.fill(entry, (byte) 0); - _available.offer(entry); + _available.release(entry); } /** diff --git a/core/java/src/net/i2p/util/TryCache.java b/core/java/src/net/i2p/util/TryCache.java index f6f8c7440..33f67db4d 100644 --- a/core/java/src/net/i2p/util/TryCache.java +++ b/core/java/src/net/i2p/util/TryCache.java @@ -11,6 +11,7 @@ import java.util.concurrent.locks.ReentrantLock; * @author zab * * @param + * @since 0.9.36 */ public class TryCache { @@ -24,9 +25,10 @@ public class TryCache { } private final ObjectFactory factory; - private final int capacity; - private final List items; - private final Lock lock = new ReentrantLock(); + protected final int capacity; + protected final List items; + protected final Lock lock = new ReentrantLock(); + protected long _lastUnderflow; /** * @param factory to be used for creating new instances @@ -47,6 +49,8 @@ public class TryCache { try { if (!items.isEmpty()) { rv = items.remove(items.size() - 1); + } else { + _lastUnderflow = System.currentTimeMillis(); } } finally { lock.unlock(); diff --git a/history.txt b/history.txt index 60ad2252c..8da436916 100644 --- a/history.txt +++ b/history.txt @@ -1,3 +1,10 @@ +2018-07-10 zzz + * Installer (ticket #1864): + - Fix wrapper selection on Windows 10 + - Add support for IzPack 5 + * SSU: Sync/notify improvements (ticket #2260) + * Util: Convert more caches to TryCache (ticket #2263) + 2018-07-08 zzz * i2psnark: Add comment icon (ticket #2278) * NTCP2: Avoid possible NPEs (ticket #2286) diff --git a/installer/install.xml b/installer/install.xml index 5112714e8..3a194d9c1 100644 --- a/installer/install.xml +++ b/installer/install.xml @@ -102,6 +102,17 @@ + + + + + + + datamodel + 64 + + + @@ -115,9 +126,9 @@ - - - + + + @@ -136,13 +147,6 @@ - - - SYSTEM_sun_arch_data_model - 64 - - - + + + + + i2p + 0.9.35 + + + + https://geti2p.net/ + 1.7 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + datamodel + 64 + + + + + + + + + + + + + + + + + + + + Base installation files + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Automatically start I2P in the background + + + + + + + + + + + + + diff --git a/installer/resources/checklist.md b/installer/resources/checklist.md index df27864ef..32fcca50b 100644 --- a/installer/resources/checklist.md +++ b/installer/resources/checklist.md @@ -106,6 +106,7 @@ 6. Change revision in: - `history.txt` - `installer/install.xml` + - `installer/install5.xml` - `core/java/src/net/i2p/CoreVersion.java` - `router/java/src/net/i2p/router/RouterVersion.java` - (change to BUILD = 0 and EXTRA = "") diff --git a/installer/resources/postinstall.sh b/installer/resources/postinstall.sh index 21a23d686..e81659de5 100644 --- a/installer/resources/postinstall.sh +++ b/installer/resources/postinstall.sh @@ -104,7 +104,7 @@ fi chmod 755 ./eepget rm -rf ./icons ./lib/wrapper -rm -f ./lib/*.dll /*.bat ./*.cmd ./*.exe ./utility.jar +rm -f ./lib/*.dll ./*.bat ./*.cmd ./*.exe ./utility.jar if [ ! `echo $HOST_OS |grep osx` ]; then rm -rf ./Start\ I2P\ Router.app diff --git a/router/java/src/com/southernstorm/noise/protocol/HandshakeState.java b/router/java/src/com/southernstorm/noise/protocol/HandshakeState.java index 377e4f1fa..df4e9f311 100644 --- a/router/java/src/com/southernstorm/noise/protocol/HandshakeState.java +++ b/router/java/src/com/southernstorm/noise/protocol/HandshakeState.java @@ -882,14 +882,6 @@ public class HandshakeState implements Destroyable { return symmetric.getChainingKey(); } - /** - * I2P for getting current hash for siphash calculation - * @return NOT a copy, do not modify - */ - public byte[] getHash() { - return symmetric.getHandshakeHash(); - } - /** * I2P debug */ diff --git a/router/java/src/net/i2p/router/RouterVersion.java b/router/java/src/net/i2p/router/RouterVersion.java index fecba78d6..7ca1a3ec2 100644 --- a/router/java/src/net/i2p/router/RouterVersion.java +++ b/router/java/src/net/i2p/router/RouterVersion.java @@ -18,7 +18,7 @@ public class RouterVersion { /** deprecated */ public final static String ID = "Monotone"; public final static String VERSION = CoreVersion.VERSION; - public final static long BUILD = 7; + public final static long BUILD = 10; /** for example "-test" */ public final static String EXTRA = ""; diff --git a/router/java/src/net/i2p/router/transport/ntcp/NTCP2Payload.java b/router/java/src/net/i2p/router/transport/ntcp/NTCP2Payload.java index 01c686dec..516aac349 100644 --- a/router/java/src/net/i2p/router/transport/ntcp/NTCP2Payload.java +++ b/router/java/src/net/i2p/router/transport/ntcp/NTCP2Payload.java @@ -343,8 +343,6 @@ class NTCP2Payload { * Big endian. * Same as DataHelper.fromLong(src, offset, 8) but allows negative result * - * Package private for NTCP2Payload. - * * @throws ArrayIndexOutOfBoundsException * @since 0.9.36 */ diff --git a/router/java/src/net/i2p/router/transport/ntcp/OutboundNTCP2State.java b/router/java/src/net/i2p/router/transport/ntcp/OutboundNTCP2State.java index 688824f4a..5295cf2e9 100644 --- a/router/java/src/net/i2p/router/transport/ntcp/OutboundNTCP2State.java +++ b/router/java/src/net/i2p/router/transport/ntcp/OutboundNTCP2State.java @@ -442,7 +442,7 @@ class OutboundNTCP2State implements EstablishState { tk = new SessionKey(temp_key); byte[] ask_master = doHMAC(ctx, tk, ASK); byte[] tmp = new byte[32 + SIPHASH.length]; - byte[] hash = state.getHash(); + byte[] hash = state.getHandshakeHash(); System.arraycopy(hash, 0, tmp, 0, 32); System.arraycopy(SIPHASH, 0, tmp, 32, SIPHASH.length); tk = new SessionKey(ask_master); diff --git a/router/java/src/net/i2p/router/transport/udp/InboundMessageFragments.java b/router/java/src/net/i2p/router/transport/udp/InboundMessageFragments.java index 592936768..27819055d 100644 --- a/router/java/src/net/i2p/router/transport/udp/InboundMessageFragments.java +++ b/router/java/src/net/i2p/router/transport/udp/InboundMessageFragments.java @@ -260,7 +260,7 @@ class InboundMessageFragments /*implements UDPTransport.PartialACKSource */{ // By calling add(), this also is a failsafe against possible // races in OutboundMessageFragments. if (newAck && from.getOutboundMessageCount() > 0) - _outbound.add(from); + _outbound.add(from, 0); return rv; } diff --git a/router/java/src/net/i2p/router/transport/udp/OutboundMessageFragments.java b/router/java/src/net/i2p/router/transport/udp/OutboundMessageFragments.java index 10e798999..8f605d8dc 100644 --- a/router/java/src/net/i2p/router/transport/udp/OutboundMessageFragments.java +++ b/router/java/src/net/i2p/router/transport/udp/OutboundMessageFragments.java @@ -51,11 +51,6 @@ class OutboundMessageFragments { */ private Iterator _iterator; - /** - * Avoid sync in add() if possible (not 100% reliable) - */ - private volatile boolean _isWaiting; - private volatile boolean _alive; private final PacketBuilder _builder; @@ -104,7 +99,7 @@ class OutboundMessageFragments { _alive = false; _activePeers.clear(); synchronized (_activePeers) { - _activePeers.notifyAll(); + _activePeers.notify(); } } @@ -165,7 +160,7 @@ class OutboundMessageFragments { // will throw IAE if peer == null OutboundMessageState state = new OutboundMessageState(_context, msg, peer); peer.add(state); - add(peer); + add(peer, state.fragmentSize(0)); } catch (IllegalArgumentException iae) { _transport.failed(msg, "Peer disconnected quickly"); return; @@ -182,7 +177,7 @@ class OutboundMessageFragments { if (peer == null) throw new RuntimeException("null peer for " + state); peer.add(state); - add(peer); + add(peer, state.fragmentSize(0)); //_context.statManager().addRateData("udp.outboundActiveCount", active, 0); } @@ -195,10 +190,15 @@ class OutboundMessageFragments { if (peer == null) throw new RuntimeException("null peer"); int sz = states.size(); + int min = peer.fragmentSize(); for (int i = 0; i < sz; i++) { - peer.add(states.get(i)); + OutboundMessageState state = states.get(i); + peer.add(state); + int fsz = state.fragmentSize(0); + if (fsz < min) + min = fsz; } - add(peer); + add(peer, min); //_context.statManager().addRateData("udp.outboundActiveCount", active, 0); } @@ -211,10 +211,10 @@ class OutboundMessageFragments { * There are larger chances of adding the PeerState "behind" where * the iterator is now... but these issues are the same as before concurrentification. * + * @param the minimum size we can send, or 0 to always notify * @since 0.8.9 */ - public void add(PeerState peer) { - boolean wasEmpty = _activePeers.isEmpty(); + public void add(PeerState peer, int size) { boolean added = _activePeers.add(peer); if (added) { if (_log.shouldLog(Log.DEBUG)) @@ -229,9 +229,9 @@ class OutboundMessageFragments { // no, this doesn't always work. // Also note that the iterator in getNextVolley may have alreay passed us, // or not reflect the addition. - if (_isWaiting || wasEmpty) { + if (added || size <= 0 || peer.getSendWindowBytesRemaining() >= size) { synchronized (_activePeers) { - _activePeers.notifyAll(); + _activePeers.notify(); } } } @@ -321,7 +321,6 @@ class OutboundMessageFragments { // if we've gone all the way through the loop, wait // ... unless nextSendDelay says we have more ready now if (states == null && peersProcessed >= _activePeers.size() && nextSendDelay > 0) { - _isWaiting = true; peersProcessed = 0; // why? we do this in the loop one at a time //finishMessages(); @@ -341,7 +340,6 @@ class OutboundMessageFragments { _log.debug("Woken up while waiting"); } } - _isWaiting = false; //} else { // if (_log.shouldLog(Log.DEBUG)) // _log.debug("dont wait: alive=" + _alive + " state = " + state); diff --git a/router/java/src/net/i2p/router/tunnel/TrivialPreprocessor.java b/router/java/src/net/i2p/router/tunnel/TrivialPreprocessor.java index 64668884e..8615c4a0b 100644 --- a/router/java/src/net/i2p/router/tunnel/TrivialPreprocessor.java +++ b/router/java/src/net/i2p/router/tunnel/TrivialPreprocessor.java @@ -34,7 +34,7 @@ class TrivialPreprocessor implements TunnelGateway.QueuePreprocessor { * (since ByteCache only maintains once instance for each size) * Used in BatchedPreprocessor; see add'l comments there */ - protected static final ByteCache _dataCache = ByteCache.getInstance(32, PREPROCESSED_SIZE); + protected static final ByteCache _dataCache = ByteCache.getInstance(512, PREPROCESSED_SIZE); public TrivialPreprocessor(RouterContext ctx) { _context = ctx;