diff --git a/build.xml b/build.xml index 9c8fa4c4f..83676a7c3 100644 --- a/build.xml +++ b/build.xml @@ -755,7 +755,7 @@ - + diff --git a/core/java/src/freenet/support/CPUInformation/CPUID.java b/core/java/src/freenet/support/CPUInformation/CPUID.java index 83aa33367..921f497dd 100644 --- a/core/java/src/freenet/support/CPUInformation/CPUID.java +++ b/core/java/src/freenet/support/CPUInformation/CPUID.java @@ -1,5 +1,6 @@ /* * Created on Jul 14, 2004 + * Updated on Jan 8, 2011 */ package freenet.support.CPUInformation; @@ -129,11 +130,26 @@ public class CPUID { CPUIDResult c = doCPUID(1); return c.EAX & 0xf; } - private static int getCPUFlags() + private static int getEDXCPUFlags() { CPUIDResult c = doCPUID(1); return c.EDX; } + private static int getECXCPUFlags() + { + CPUIDResult c = doCPUID(1); + return c.ECX; + } + private static int getExtendedEDXCPUFlags() + { + CPUIDResult c = doCPUID(0x80000001); + return c.EDX; + } + private static int getExtendedECXCPUFlags() + { + CPUIDResult c = doCPUID(0x80000001); + return c.ECX; + } //Returns a CPUInfo item for the current type of CPU //If I could I would declare this method in a interface named @@ -163,13 +179,25 @@ public class CPUID { return getCPUVendorID(); } public boolean hasMMX(){ - return (getCPUFlags() & 0x800000) >0; //Bit 23 + return (getEDXCPUFlags() & 0x800000) >0; //EDX Bit 23 } public boolean hasSSE(){ - return (getCPUFlags() & 0x2000000) >0; //Bit 25 + return (getEDXCPUFlags() & 0x2000000) >0; //EDX Bit 25 } public boolean hasSSE2(){ - return (getCPUFlags() & 0x4000000) >0; //Bit 26 + return (getEDXCPUFlags() & 0x4000000) >0; //EDX Bit 26 + } + public boolean hasSSE3(){ + return (getEDXCPUFlags() & 0x1) >0; //ECX Bit 0 + } + public boolean hasSSE41(){ + return (getEDXCPUFlags() & 0x80000) >0; //ECX Bit 19 + } + public boolean hasSSE42(){ + return (getEDXCPUFlags() & 0x100000) >0; //ECX Bit 20 + } + public boolean hasSSE4A(){ + return (getExtendedECXCPUFlags() & 0x40) >0; //Extended ECX Bit 6 } public boolean IsC3Compatible() { return false; } } @@ -296,77 +324,119 @@ public class CPUID { return getCPUFamily() > 6 || (getCPUFamily() == 6 && getCPUModel() >=3); } public boolean IsPentium3Compatible() - { - return getCPUFamily() > 6 || (getCPUFamily() == 6 && getCPUModel() >=7); + { + // Atom + if (getCPUExtendedModel() == 1 && (getCPUFamily() == 6 && (getCPUModel() == 12))){ + return true; + // ?? + } else if (getCPUExtendedModel() == 0 && (getCPUFamily() > 6 || (getCPUFamily() == 6 && getCPUModel() >=7))){ + return true; + } else { + return false; + } } public boolean IsPentium4Compatible() - { - return getCPUFamily() >= 15; + { + // P4 + if (getCPUFamily() >= 15){ + return true; + // Xeon MP (45nm) or Core i7 + } else if (getCPUExtendedModel() == 1 && (getCPUFamily() == 6 && (getCPUModel() == 10 || getCPUModel() == 13))){ + return true; + // Core 2 Duo + } else if (getCPUExtendedModel() == 0 && getCPUFamily() == 6 && getCPUModel() == 15){ + return true; + } else { + return false; + } } public String getCPUModelString() throws UnknownCPUException { - if(getCPUFamily() == 4){ - switch(getCPUModel()){ - case 0: - return "486 DX-25/33"; - case 1: - return "486 DX-50"; - case 2: - return "486 SX"; - case 3: - return "486 DX/2"; - case 4: - return "486 SL"; - case 5: - return "486 SX/2"; - case 7: - return "486 DX/2-WB"; - case 8: - return "486 DX/4"; - case 9: - return "486 DX/4-WB"; - } - } - if(getCPUFamily() == 5){ - switch(getCPUModel()){ - case 0: - return "Pentium 60/66 A-step"; - case 1: - return "Pentium 60/66"; - case 2: - return "Pentium 75 - 200"; - case 3: - return "OverDrive PODP5V83"; - case 4: - return "Pentium MMX"; - case 7: - return "Mobile Pentium 75 - 200"; - case 8: - return "Mobile Pentium MMX"; - } + if (getCPUExtendedModel() == 0){ + if(getCPUFamily() == 4){ + switch(getCPUModel()){ + case 0: + return "486 DX-25/33"; + case 1: + return "486 DX-50"; + case 2: + return "486 SX"; + case 3: + return "486 DX/2"; + case 4: + return "486 SL"; + case 5: + return "486 SX/2"; + case 7: + return "486 DX/2-WB"; + case 8: + return "486 DX/4"; + case 9: + return "486 DX/4-WB"; + } + } + } + if (getCPUExtendedModel() == 0){ + if(getCPUFamily() == 5){ + switch(getCPUModel()){ + case 0: + return "Pentium 60/66 A-step"; + case 1: + return "Pentium 60/66"; + case 2: + return "Pentium 75 - 200"; + case 3: + return "OverDrive PODP5V83"; + case 4: + return "Pentium MMX"; + case 7: + return "Mobile Pentium 75 - 200"; + case 8: + return "Mobile Pentium MMX"; + } + } } if(getCPUFamily() == 6){ - switch(getCPUModel()){ - case 0: - return "Pentium Pro A-step"; - case 1: - return "Pentium Pro"; - case 3: - return "Pentium II (Klamath)"; - case 5: - return "Pentium II (Deschutes), Celeron (Covington), Mobile Pentium II (Dixon)"; - case 6: - return "Mobile Pentium II, Celeron (Mendocino)"; - case 7: - return "Pentium III (Katmai)"; - case 8: - return "Pentium III (Coppermine), Celeron w/SSE"; - case 9: - return "Mobile Pentium III"; - case 10: - return "Pentium III Xeon (Cascades)"; - case 11: - return "Pentium III (130 nm)"; - } + if (getCPUExtendedModel() == 0){ + switch(getCPUModel()){ + case 0: + return "Pentium Pro A-step"; + case 1: + return "Pentium Pro"; + case 3: + return "Pentium II (Klamath)"; + case 5: + return "Pentium II (Deschutes), Celeron (Covington), Mobile Pentium II (Dixon)"; + case 6: + return "Mobile Pentium II, Celeron (Mendocino)"; + case 7: + return "Pentium III (Katmai)"; + case 8: + return "Pentium III (Coppermine), Celeron w/SSE"; + case 9: + return "Mobile Pentium III (Banias)"; + case 10: + return "Pentium III Xeon (Cascades)"; + case 11: + return "Pentium III (130 nm)"; + case 13: + return "Mobile Pentium III (Dothan)"; + case 14: + return "Mobile Core (Yonah)"; + case 15: + return "Core 2 (Conroe)"; + } + } else { + if (getCPUExtendedModel() == 1){ + switch(getCPUModel()){ + case 10: + return "Core i7"; + case 12: + return "Atom"; + case 13: + return "Xeon MP"; + } + } + } } if(getCPUFamily() == 7){ switch(getCPUModel()){ @@ -384,6 +454,10 @@ public class CPUID { return "Pentium IV (130 nm)"; case 3: return "Pentium IV (90 nm)"; + case 4: + return "Pentium IV (90 nm)"; + case 6: + return "Pentium IV (65 nm)"; } } if(getCPUExtendedFamily() == 1){ @@ -407,7 +481,7 @@ public class CPUID { System.out.println("CPU Family: " + getCPUFamily()); System.out.println("CPU Model: " + getCPUModel()); System.out.println("CPU Stepping: " + getCPUStepping()); - System.out.println("CPU Flags: " + getCPUFlags()); + System.out.println("CPU Flags: " + getEDXCPUFlags()); CPUInfo c = getInfo(); System.out.println(" **More CPUInfo**"); @@ -415,6 +489,10 @@ public class CPUID { System.out.println(" CPU has MMX: " + c.hasMMX()); System.out.println(" CPU has SSE: " + c.hasSSE()); System.out.println(" CPU has SSE2: " + c.hasSSE2()); + System.out.println(" CPU has SSE3: " + c.hasSSE3()); + System.out.println(" CPU has SSE4.1: " + c.hasSSE41()); + System.out.println(" CPU has SSE4.2: " + c.hasSSE42()); + System.out.println(" CPU has SSE4A: " + c.hasSSE4A()); if(c instanceof IntelCPUInfo){ System.out.println(" **Intel-info**"); System.out.println(" Is pII-compatible: "+((IntelCPUInfo)c).IsPentium2Compatible()); diff --git a/core/java/src/freenet/support/CPUInformation/CPUInfo.java b/core/java/src/freenet/support/CPUInformation/CPUInfo.java index e6b660927..43dfcd8d0 100644 --- a/core/java/src/freenet/support/CPUInformation/CPUInfo.java +++ b/core/java/src/freenet/support/CPUInformation/CPUInfo.java @@ -1,6 +1,6 @@ /* - * Created on Jul 16, 2004 - * + * Created on Jul 14, 2004 + * Updated on Jan 8, 2011 */ package freenet.support.CPUInformation; @@ -42,5 +42,25 @@ public interface CPUInfo */ public boolean hasSSE2(); + /** + * @return true iff the CPU support the SSE3 instruction set. + */ + public boolean hasSSE3(); + + /** + * @return true iff the CPU support the SSE4.1 instruction set. + */ + public boolean hasSSE41(); + + /** + * @return true iff the CPU support the SSE4.2 instruction set. + */ + public boolean hasSSE42(); + + /** + * @return true iff the CPU support the SSE4A instruction set. + */ + public boolean hasSSE4A(); + public boolean IsC3Compatible(); } diff --git a/core/java/src/net/i2p/crypto/SHA256Generator.java b/core/java/src/net/i2p/crypto/SHA256Generator.java index 8d5b6957b..87955646b 100644 --- a/core/java/src/net/i2p/crypto/SHA256Generator.java +++ b/core/java/src/net/i2p/crypto/SHA256Generator.java @@ -23,13 +23,18 @@ public final class SHA256Generator { return I2PAppContext.getGlobalContext().sha(); } - /** Calculate the SHA-256 has of the source + /** + * Calculate the SHA-256 hash of the source and cache the result. * @param source what to hash * @return hash of the source */ public final Hash calculateHash(byte[] source) { return calculateHash(source, 0, source.length); } + + /** + * Calculate the hash and cache the result. + */ public final Hash calculateHash(byte[] source, int start, int len) { Sha256Standalone digest = acquireGnu(); digest.update(source, start, len); @@ -39,6 +44,10 @@ public final class SHA256Generator { return Hash.create(rv); } + /** + * Use this if you only need the data, not a Hash object. + * Does not cache. + */ public final void calculateHash(byte[] source, int start, int len, byte out[], int outOffset) { Sha256Standalone digest = acquireGnu(); digest.update(source, start, len); diff --git a/core/java/src/net/i2p/data/DataHelper.java b/core/java/src/net/i2p/data/DataHelper.java index 33264edf2..69b315a1a 100644 --- a/core/java/src/net/i2p/data/DataHelper.java +++ b/core/java/src/net/i2p/data/DataHelper.java @@ -31,6 +31,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.Iterator; import java.util.List; @@ -1040,25 +1041,45 @@ public class DataHelper { } /** - * Sort based on the Hash of the DataStructure + * Sort based on the Hash of the DataStructure. * Warning - relatively slow. - * Only used by RouterInfo - * Why? Just because it has to be consistent so signing will work? + * WARNING - this sort order must be consistent network-wide, so while the order is arbitrary, + * it cannot be changed. + * Why? Just because it has to be consistent so signing will work. * How to spec as returning the same type as the param? + * DEPRECATED - Only used by RouterInfo. */ public static List sortStructures(Collection dataStructures) { if (dataStructures == null) return Collections.EMPTY_LIST; - ArrayList rv = new ArrayList(dataStructures.size()); - TreeMap tm = new TreeMap(); - for (DataStructure struct : dataStructures) { - tm.put(struct.calculateHash().toString(), struct); - } - for (DataStructure struct : tm.values()) { - rv.add(struct); - } + + // This used to use Hash.toString(), which is insane, since a change to toString() + // would break the whole network. Now use Hash.toBase64(). + // Note that the Base64 sort order is NOT the same as the raw byte sort order, + // despite what you may read elsewhere. + + //ArrayList rv = new ArrayList(dataStructures.size()); + //TreeMap tm = new TreeMap(); + //for (DataStructure struct : dataStructures) { + // tm.put(struct.calculateHash().toString(), struct); + //} + //for (DataStructure struct : tm.values()) { + // rv.add(struct); + //} + ArrayList rv = new ArrayList(dataStructures); + Collections.sort(rv, new DataStructureComparator()); return rv; } + /** + * See sortStructures() comments. + * @since 0.8.3 + */ + private static class DataStructureComparator implements Comparator { + public int compare(DataStructure l, DataStructure r) { + return l.calculateHash().toBase64().compareTo(r.calculateHash().toBase64()); + } + } + /** * NOTE: formatDuration2() recommended in most cases for readability */ diff --git a/core/java/src/net/i2p/data/PrivateKey.java b/core/java/src/net/i2p/data/PrivateKey.java index bf415bdc9..5b0a5e4bb 100644 --- a/core/java/src/net/i2p/data/PrivateKey.java +++ b/core/java/src/net/i2p/data/PrivateKey.java @@ -16,6 +16,9 @@ import net.i2p.crypto.KeyGenerator; * A private key is 256byte Integer. The private key represents only the * exponent, not the primes, which are constant and defined in the crypto spec. * + * Note that we use short exponents, so all but the last 28.25 bytes are zero. + * See http://www.i2p2.i2p/how_cryptography for details. + * * @author jrandom */ public class PrivateKey extends SimpleDataStructure { @@ -50,4 +53,24 @@ public class PrivateKey extends SimpleDataStructure { return KeyGenerator.getPublicKey(this); } + /** + * We assume the data has enough randomness in it, so use the last 4 bytes for speed. + * Overridden since we use short exponents, so the first 227 bytes are all zero. + * Not that we are storing PrivateKeys in any Sets or Maps anywhere. + */ + @Override + public int hashCode() { + if (_data == null) + return 0; + int rv = _data[KEYSIZE_BYTES - 4]; + for (int i = 1; i < 4; i++) + rv ^= (_data[i + (KEYSIZE_BYTES - 4)] << (i*8)); + return rv; + } + + @Override + public boolean equals(Object obj) { + if ((obj == null) || !(obj instanceof PrivateKey)) return false; + return DataHelper.eq(_data, ((PrivateKey) obj)._data); + } } diff --git a/core/java/src/net/i2p/data/RouterAddress.java b/core/java/src/net/i2p/data/RouterAddress.java index 1b4d6438f..647104574 100644 --- a/core/java/src/net/i2p/data/RouterAddress.java +++ b/core/java/src/net/i2p/data/RouterAddress.java @@ -131,10 +131,13 @@ public class RouterAddress extends DataStructureImpl { && DataHelper.eq(_transportStyle, addr.getTransportStyle()); } - /** the style should be sufficient, for speed */ + /** + * Just use style and hashCode for speed (expiration is always null). + * If we add multiple addresses of the same style, this may need to be changed. + */ @Override public int hashCode() { - return DataHelper.hashCode(_transportStyle); + return DataHelper.hashCode(_transportStyle) ^ _cost; } /** diff --git a/core/java/src/net/i2p/data/RouterInfo.java b/core/java/src/net/i2p/data/RouterInfo.java index 9c6db01f9..b4aae94b1 100644 --- a/core/java/src/net/i2p/data/RouterInfo.java +++ b/core/java/src/net/i2p/data/RouterInfo.java @@ -14,6 +14,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.Collection; +import java.util.Collections; import java.util.Date; import java.util.HashSet; import java.util.Iterator; @@ -39,7 +40,8 @@ public class RouterInfo extends DatabaseEntry { private RouterIdentity _identity; private volatile long _published; private final Set _addresses; - private final Set _peers; + /** may be null to save memory, no longer final */ + private Set _peers; private /* FIXME final FIXME */ Properties _options; private volatile boolean _validated; private volatile boolean _isValid; @@ -47,6 +49,10 @@ public class RouterInfo extends DatabaseEntry { private volatile byte _byteified[]; private volatile int _hashCode; private volatile boolean _hashCodeInitialized; + /** should we cache the byte and string versions _byteified ? **/ + private boolean _shouldCache; + /** maybe we should check if we are floodfill? */ + private static final boolean CACHE_ALL = Runtime.getRuntime().maxMemory() > 128*1024*1024l; public static final String PROP_NETWORK_ID = "netId"; public static final String PROP_CAPABILITIES = "caps"; @@ -58,7 +64,6 @@ public class RouterInfo extends DatabaseEntry { public RouterInfo() { _addresses = new HashSet(2); - _peers = new HashSet(0); _options = new OrderedProperties(); } @@ -70,6 +75,7 @@ public class RouterInfo extends DatabaseEntry { setPeers(old.getPeers()); setOptions(old.getOptions()); setSignature(old.getSignature()); + // copy over _byteified? } public long getDate() { @@ -105,6 +111,11 @@ public class RouterInfo extends DatabaseEntry { public void setIdentity(RouterIdentity ident) { _identity = ident; resetCache(); + // We only want to cache the bytes for our own RI, which is frequently written. + // To cache for all RIs doubles the RI memory usage. + // setIdentity() is only called when we are creating our own RI. + // Otherwise, the data is populated with readBytes(). + _shouldCache = true; } /** @@ -159,6 +170,8 @@ public class RouterInfo extends DatabaseEntry { * @deprecated Implemented here but unused elsewhere */ public Set getPeers() { + if (_peers == null) + return Collections.EMPTY_SET; return _peers; } @@ -169,9 +182,15 @@ public class RouterInfo extends DatabaseEntry { * @deprecated Implemented here but unused elsewhere */ public void setPeers(Set peers) { + if (peers == null || peers.isEmpty()) { + _peers = null; + return; + } + if (_peers == null) + _peers = new HashSet(2); synchronized (_peers) { _peers.clear(); - if (peers != null) _peers.addAll(peers); + _peers.addAll(peers); } resetCache(); } @@ -223,7 +242,6 @@ public class RouterInfo extends DatabaseEntry { if (_byteified != null) return _byteified; if (_identity == null) throw new DataFormatException("Router identity isn't set? wtf!"); if (_addresses == null) throw new DataFormatException("Router addressess isn't set? wtf!"); - if (_peers == null) throw new DataFormatException("Router peers isn't set? wtf!"); if (_options == null) throw new DataFormatException("Router options isn't set? wtf!"); long before = Clock.getInstance().now(); @@ -239,6 +257,9 @@ public class RouterInfo extends DatabaseEntry { DataHelper.writeLong(out, 1, sz); Collection addresses = _addresses; if (sz > 1) + // WARNING this sort algorithm cannot be changed, as it must be consistent + // network-wide. The signature is not checked at readin time, but only + // later, and the addresses are stored in a Set, not a List. addresses = (Collection) DataHelper.sortStructures(addresses); for (RouterAddress addr : addresses) { addr.writeBytes(out); @@ -248,12 +269,14 @@ public class RouterInfo extends DatabaseEntry { // answer: they're always empty... they're a placeholder for one particular // method of trusted links, which isn't implemented in the router // at the moment, and may not be later. - // fixme to reduce objects - allow _peers == null - int psz = _peers.size(); + int psz = _peers == null ? 0 : _peers.size(); DataHelper.writeLong(out, 1, psz); if (psz > 0) { Collection peers = _peers; if (psz > 1) + // WARNING this sort algorithm cannot be changed, as it must be consistent + // network-wide. The signature is not checked at readin time, but only + // later, and the hashes are stored in a Set, not a List. peers = (Collection) DataHelper.sortStructures(peers); for (Hash peerHash : peers) { peerHash.writeBytes(out); @@ -266,7 +289,8 @@ public class RouterInfo extends DatabaseEntry { byte data[] = out.toByteArray(); long after = Clock.getInstance().now(); _log.debug("getBytes() took " + (after - before) + "ms"); - _byteified = data; + if (CACHE_ALL || _shouldCache) + _byteified = data; return data; } @@ -466,10 +490,15 @@ public class RouterInfo extends DatabaseEntry { _addresses.add(address); } int numPeers = (int) DataHelper.readLong(in, 1); - for (int i = 0; i < numPeers; i++) { - Hash peerIdentityHash = new Hash(); - peerIdentityHash.readBytes(in); - _peers.add(peerIdentityHash); + if (numPeers == 0) { + _peers = null; + } else { + _peers = new HashSet(numPeers); + for (int i = 0; i < numPeers; i++) { + Hash peerIdentityHash = new Hash(); + peerIdentityHash.readBytes(in); + _peers.add(peerIdentityHash); + } } _options = DataHelper.readProperties(in); _signature = new Signature(); @@ -504,7 +533,7 @@ public class RouterInfo extends DatabaseEntry { && _published == info.getPublished() && DataHelper.eq(_addresses, info.getAddresses()) && DataHelper.eq(_options, info.getOptions()) - && DataHelper.eq(_peers, info.getPeers()); + && DataHelper.eq(getPeers(), info.getPeers()); } @Override @@ -530,7 +559,7 @@ public class RouterInfo extends DatabaseEntry { RouterAddress addr = (RouterAddress) iter.next(); buf.append("\n\t\tAddress: ").append(addr); } - Set peers = _peers; // getPeers() + Set peers = getPeers(); buf.append("\n\tPeers: #: ").append(peers.size()); for (Iterator iter = peers.iterator(); iter.hasNext();) { Hash hash = (Hash) iter.next(); diff --git a/core/java/src/net/i2p/util/LogManager.java b/core/java/src/net/i2p/util/LogManager.java index 8a9549724..be6fe2ef9 100644 --- a/core/java/src/net/i2p/util/LogManager.java +++ b/core/java/src/net/i2p/util/LogManager.java @@ -152,7 +152,9 @@ public class LogManager { if (_writer != null) return; _writer = new LogWriter(this); - Thread t = new I2PThread(_writer, "LogWriter", true); + // NOT an I2PThread, as it contains logging and we end up with problems + Thread t = new Thread(_writer, "LogWriter"); + t.setDaemon(true); t.start(); } diff --git a/history.txt b/history.txt index 894223bdd..118de8648 100644 --- a/history.txt +++ b/history.txt @@ -1,3 +1,16 @@ +2011-01-09 zzz + * DataHelper: Speed up and annotate sortStructures() + * Data Structures: More caching improvements, don't cache where we shouldn't + * NetDB: Don't rescan netDb directory unless changed, + to reduce Hash cache thrash (backport from test4) + * RouterInfo: + - Don't cache byteified data by default, to save ~1.5 MB + - Don't create empty peers Set, to save ~100KB + +2011-01-07 zzz + * Data Structures: More caching + * i2psnark: Improve request tracking to reduce memory usage + 2011-01-05 zzz * build.xml: Fix findbugs target * Fix some miscellaneous findbugs bugs diff --git a/installer/resources/readme/readme.html b/installer/resources/readme/readme.html index 00667aea2..e6cfe69a8 100644 --- a/installer/resources/readme/readme.html +++ b/installer/resources/readme/readme.html @@ -15,7 +15,7 @@

Welcome to the Invisible Internet

+
  • Network integration
    The first time you start I2P it may take a few minutes to bootstrap (integrate) you into the network and find additional peers to optimize your integration, so please be patient. When I2P starts up, and during normal operation, I2P's tunnel build readiness indicator (immediately above the Local Destinations section in the sidepanel) may tell you that I2P is "Rejecting Tunnels"; this is normal behavior and should be of no cause for concern! Once green stars are indicated next to your Local Destinations, there is a wide variety of things you can do with I2P, and below we introduce you to some of them.
  • Services on I2P