loveisgrief/i2p.i2p
1
0
Fork 0
forked from I2P_Developers/i2p.i2p
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* IP Lookup:

Browse Source 
- Add caching in RouterAddress with secondary caching
      in Addresses; use caches to speed up transport bids,
      blocklist checks, geoip lookups, and profile organizer checks
      (ticket #707)
    - Limit IP cache size in TransportImpl
    - Clear caches at shutdown
  * RouterAddress: Remove unused expiration field to save space
This commit is contained in:
zzz
2012-09-04 20:33:04 +00:00
parent 2cddf1405f
commit 9286d6a7b8
13 changed files with 233 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
core/java/src/net/i2p/data/RouterAddress.java
View File

@@ -17,6 +17,7 @@ import java.util.Date;
import java.util.Map;
import java.util.Properties;
import net.i2p.util.Addresses;
import net.i2p.util.OrderedProperties;
/**
@@ -33,9 +34,15 @@ import net.i2p.util.OrderedProperties;
*/
public class RouterAddress extends DataStructureImpl {
private int _cost;
private Date _expiration;
//private Date _expiration;
private String _transportStyle;
private final Properties _options;
// cached values
private byte[] _ip;
private int _port;
public static final String PROP_HOST = "host";
public static final String PROP_PORT = "port";
public RouterAddress() {
_cost = -1;
@@ -68,18 +75,21 @@ public class RouterAddress extends DataStructureImpl {
* is null, then the address never expires.
*
* @deprecated unused for now
* @return null always
*/
public Date getExpiration() {
return _expiration;
//return _expiration;
return null;
}
/**
* Configure the expiration date of the address (null for no expiration)
*
* Unused for now, always null
* @deprecated unused for now
*/
public void setExpiration(Date expiration) {
_expiration = expiration;
//_expiration = expiration;
}
/**
@@ -140,6 +150,51 @@ public class RouterAddress extends DataStructureImpl {
_options.putAll(options);
}
/**
* Caching version of InetAddress.getByName(getOption("host")).getAddress(), which is slow.
* Caches numeric host names only.
* Will resolve but not cache resolution of DNS host names.
*
* @return IP or null
* @since 0.9.3
*/
public byte[] getIP() {
if (_ip != null)
return _ip;
byte[] rv = null;
String host = _options.getProperty(PROP_HOST);
if (host != null) {
rv = Addresses.getIP(host);
if (rv != null &&
(host.replaceAll("[0-9\\.]", "").length() == 0 ||
host.replaceAll("[0-9a-fA-F:]", "").length() == 0)) {
_ip = rv;
}
}
return rv;
}
/**
* Caching version of Integer.parseInt(getOption("port"))
* Caches valid ports 1-65535 only.
*
* @return 1-65535 or 0 if invalid
* @since 0.9.3
*/
public int getPort() {
if (_port != 0)
return _port;
String port = _options.getProperty(PROP_PORT);
if (port != null) {
try {
int rv = Integer.parseInt(port);
if (rv > 0 && rv <= 65535)
_port = rv;
} catch (NumberFormatException nfe) {}
}
return _port;
}
/**
* @throws IllegalStateException if was already read in
*/
@@ -147,7 +202,8 @@ public class RouterAddress extends DataStructureImpl {
if (_transportStyle != null)
throw new IllegalStateException();
_cost = (int) DataHelper.readLong(in, 1);
_expiration = DataHelper.readDate(in);
//_expiration = DataHelper.readDate(in);
DataHelper.readDate(in);
_transportStyle = DataHelper.readString(in);
// reduce Object proliferation
if (_transportStyle.equals("SSU"))
@@ -161,7 +217,8 @@ public class RouterAddress extends DataStructureImpl {
if ((_cost < 0) || (_transportStyle == null))
throw new DataFormatException("Not enough data to write a router address");
DataHelper.writeLong(out, 1, _cost);
DataHelper.writeDate(out, _expiration);
//DataHelper.writeDate(out, _expiration);
DataHelper.writeDate(out, null);
DataHelper.writeString(out, _transportStyle);
DataHelper.writeProperties(out, _options);
}
@@ -198,15 +255,13 @@ public class RouterAddress extends DataStructureImpl {
buf.append("[RouterAddress: ");
buf.append("\n\tTransportStyle: ").append(_transportStyle);
buf.append("\n\tCost: ").append(_cost);
buf.append("\n\tExpiration: ").append(_expiration);
if (_options != null) {
//buf.append("\n\tExpiration: ").append(_expiration);
buf.append("\n\tOptions: #: ").append(_options.size());
for (Map.Entry e : _options.entrySet()) {
String key = (String) e.getKey();
String val = (String) e.getValue();
buf.append("\n\t\t[").append(key).append("] = [").append(val).append("]");
}
}
buf.append("]");
return buf.toString();
}

66
core/java/src/net/i2p/util/Addresses.java
View File

@@ -10,10 +10,13 @@ import java.net.NetworkInterface;
import java.net.SocketException;
import java.net.UnknownHostException;
import java.util.Enumeration;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import net.i2p.I2PAppContext;
/**
* Methods to get the local addresses, and other IP utilities
@@ -152,6 +155,69 @@ public abstract class Addresses {
}
}
/**
* Textual IP to bytes, because InetAddress.getByName() is slow.
*
* @since 0.9.3
*/
private static final Map<String, byte[]> _IPAddress;
static {
int size;
I2PAppContext ctx = I2PAppContext.getCurrentContext();
if (ctx != null && ctx.isRouterContext()) {
long maxMemory = Runtime.getRuntime().maxMemory();
if (maxMemory == Long.MAX_VALUE)
maxMemory = 96*1024*1024l;
long min = 128;
long max = 4096;
// 512 nominal for 128 MB
size = (int) Math.max(min, Math.min(max, 1 + (maxMemory / (256*1024))));
} else {
size = 32;
}
_IPAddress = new LHMCache(size);
}
/**
* Caching version of InetAddress.getByName(host).getAddress(), which is slow.
* Caches numeric host names only.
* Will resolve but not cache DNS host names.
*
* @param host DNS or IPv4 or IPv6 host name; if null returns null
* @return IP or null
* @since 0.9.3
*/
public static byte[] getIP(String host) {
if (host == null)
return null;
byte[] rv;
synchronized (_IPAddress) {
rv = _IPAddress.get(host);
}
if (rv == null) {
try {
rv = InetAddress.getByName(host).getAddress();
if (host.replaceAll("[0-9\\.]", "").length() == 0 ||
host.replaceAll("[0-9a-fA-F:]", "").length() == 0) {
synchronized (_IPAddress) {
_IPAddress.put(host, rv);
}
}
} catch (UnknownHostException uhe) {}
}
return rv;
}
/**
* @since 0.9.3
*/
public static void clearCaches() {
synchronized(_IPAddress) {
_IPAddress.clear();
}
}
/**
* Print out the local addresses
*/

39
router/java/src/net/i2p/router/Blocklist.java
View File

@@ -24,6 +24,7 @@ import java.util.Set;
import java.util.TreeSet;
import net.i2p.data.Base64;
import net.i2p.data.DataHelper;
import net.i2p.data.Hash;
import net.i2p.data.RouterAddress;
import net.i2p.data.RouterInfo;
@@ -438,14 +439,8 @@ public class Blocklist {
* of IP ranges read in from the file.
*/
public void add(String ip) {
InetAddress pi;
try {
pi = InetAddress.getByName(ip);
} catch (UnknownHostException uhe) {
return;
}
if (pi == null) return;
byte[] pib = pi.getAddress();
byte[] pib = Addresses.getIP(ip);
if (pib == null) return;
add(pib);
}
@@ -478,21 +473,13 @@ public class Blocklist {
List<byte[]> rv = new ArrayList(1);
RouterInfo pinfo = _context.netDb().lookupRouterInfoLocally(peer);
if (pinfo == null) return rv;
String oldphost = null;
byte[] oldpib = null;
// for each peer address
for (RouterAddress pa : pinfo.getAddresses()) {
String phost = pa.getOption("host");
if (phost == null) continue;
if (oldphost != null && oldphost.equals(phost)) continue;
oldphost = phost;
InetAddress pi;
try {
pi = InetAddress.getByName(phost);
} catch (UnknownHostException uhe) {
continue;
}
if (pi == null) continue;
byte[] pib = pi.getAddress();
byte[] pib = pa.getIP();
if (pib == null) continue;
if (DataHelper.eq(oldpib, pib)) continue;
oldpib = pib;
rv.add(pib);
}
return rv;
@@ -520,14 +507,8 @@ public class Blocklist {
* calling this externally won't shitlist the peer, this is just an IP check
*/
public boolean isBlocklisted(String ip) {
InetAddress pi;
try {
pi = InetAddress.getByName(ip);
} catch (UnknownHostException uhe) {
return false;
}
if (pi == null) return false;
byte[] pib = pi.getAddress();
byte[] pib = Addresses.getIP(ip);
if (pib == null) return false;
return isBlocklisted(pib);
}

12
router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java
View File

@@ -1261,16 +1261,8 @@ public class ProfileOrganizer {
if (paddr == null)
return rv;
for (RouterAddress pa : paddr) {
String phost = pa.getOption("host");
if (phost == null) continue;
InetAddress pi;
try {
pi = InetAddress.getByName(phost);
} catch (UnknownHostException uhe) {
continue;
}
if (pi == null) continue;
byte[] pib = pi.getAddress();
byte[] pib = pa.getIP();
if (pib == null) continue;
rv.add(maskedIP(pib, mask));
}
return rv;

3
router/java/src/net/i2p/router/transport/CommSystemFacadeImpl.java
View File

@@ -62,6 +62,7 @@ public class CommSystemFacadeImpl extends CommSystemFacade {
public void shutdown() {
if (_manager != null)
_manager.shutdown();
_geoIP.shutdown();
}
public void restart() {
@@ -250,7 +251,7 @@ public class CommSystemFacadeImpl extends CommSystemFacade {
props.setProperty(NTCPAddress.PROP_PORT, port);
RouterAddress addr = new RouterAddress();
addr.setCost(NTCPAddress.DEFAULT_COST);
addr.setExpiration(null);
//addr.setExpiration(null);
addr.setOptions(props);
addr.setTransportStyle(NTCPTransport.STYLE);
//if (isNew) {

32
router/java/src/net/i2p/router/transport/GeoIP.java
View File

@@ -21,6 +21,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import net.i2p.data.Hash;
import net.i2p.router.Router;
import net.i2p.router.RouterContext;
import net.i2p.util.Addresses;
import net.i2p.util.ConcurrentHashSet;
import net.i2p.util.Log;
@@ -72,6 +73,17 @@ class GeoIP {
static final String COUNTRY_FILE_DEFAULT = "countries.txt";
public static final String PROP_IP_COUNTRY = "i2np.lastCountry";
/**
* @since 0.9.3
*/
public void shutdown() {
_codeToName.clear();
_codeCache.clear();
_IPToCountry.clear();
_pendingSearch.clear();
_notFound.clear();
}
/**
* Fire off a thread to lookup all pending IPs.
* There is no indication of completion.
@@ -297,14 +309,8 @@ class GeoIP {
* Add to the list needing lookup
*/
public void add(String ip) {
InetAddress pi;
try {
pi = InetAddress.getByName(ip);
} catch (UnknownHostException uhe) {
return;
}
if (pi == null) return;
byte[] pib = pi.getAddress();
byte[] pib = Addresses.getIP(ip);
if (pib == null) return;
add(pib);
}
@@ -325,14 +331,8 @@ class GeoIP {
* @return lower-case code, generally two letters, or null.
*/
public String get(String ip) {
InetAddress pi;
try {
pi = InetAddress.getByName(ip);
} catch (UnknownHostException uhe) {
return null;
}
if (pi == null) return null;
byte[] pib = pi.getAddress();
byte[] pib = Addresses.getIP(ip);
if (pib == null) return null;
return get(pib);
}

36
router/java/src/net/i2p/router/transport/TransportImpl.java
View File

@@ -22,6 +22,7 @@ import java.util.Set;
import java.util.Vector;
import java.util.concurrent.ConcurrentHashMap;
import net.i2p.data.DataHelper;
import net.i2p.data.Hash;
import net.i2p.data.RouterAddress;
import net.i2p.data.RouterIdentity;
@@ -35,6 +36,7 @@ import net.i2p.router.Router;
import net.i2p.router.RouterContext;
import net.i2p.router.networkdb.kademlia.FloodfillNetworkDatabaseFacade;
import net.i2p.util.ConcurrentHashSet;
import net.i2p.util.LHMCache;
import net.i2p.util.Log;
import net.i2p.util.SimpleScheduler;
import net.i2p.util.SimpleTimer;
@@ -53,7 +55,18 @@ public abstract class TransportImpl implements Transport {
private final Map<Hash, Long> _unreachableEntries;
private final Set<Hash> _wasUnreachableEntries;
/** global router ident -> IP */
private static final Map<Hash, byte[]> _IPMap = new ConcurrentHashMap(128);
private static final Map<Hash, byte[]> _IPMap;
static {
long maxMemory = Runtime.getRuntime().maxMemory();
if (maxMemory == Long.MAX_VALUE)
maxMemory = 96*1024*1024l;
long min = 512;
long max = 4096;
// 1024 nominal for 128 MB
int size = (int) Math.max(min, Math.min(max, 1 + (maxMemory / (128*1024))));
_IPMap = new LHMCache(size);
}
/**
* Initialize the new transport
@@ -585,12 +598,27 @@ public abstract class TransportImpl implements Transport {
}
public void setIP(Hash peer, byte[] ip) {
_IPMap.put(peer, ip);
_context.commSystem().queueLookup(ip);
byte[] old;
synchronized (_IPMap) {
old = _IPMap.put(peer, ip);
}
if (!DataHelper.eq(old, ip))
_context.commSystem().queueLookup(ip);
}
public static byte[] getIP(Hash peer) {
return _IPMap.get(peer);
synchronized (_IPMap) {
return _IPMap.get(peer);
}
}
/**
* @since 0.9.3
*/
static void clearCaches() {
synchronized(_IPMap) {
_IPMap.clear();
}
}
/** @param addr non-null */

2
router/java/src/net/i2p/router/transport/TransportManager.java
View File

@@ -185,6 +185,8 @@ public class TransportManager implements TransportEventListener {
public void shutdown() {
stopListening();
_dhThread.shutdown();
Addresses.clearCaches();
TransportImpl.clearCaches();
}
public Transport getTransport(String style) {

46
router/java/src/net/i2p/router/transport/ntcp/NTCPAddress.java
View File

@@ -8,13 +8,13 @@ package net.i2p.router.transport.ntcp;
*
*/
import java.net.InetAddress;
import java.util.Properties;
import net.i2p.I2PAppContext;
import net.i2p.data.DataHelper;
import net.i2p.data.RouterAddress;
import net.i2p.router.transport.TransportImpl;
import net.i2p.util.Addresses;
import net.i2p.util.Log;
/**
@@ -25,9 +25,9 @@ public class NTCPAddress {
private final String _host;
//private InetAddress _addr;
/** Port number used in RouterAddress definitions */
public final static String PROP_PORT = "port";
public final static String PROP_PORT = RouterAddress.PROP_PORT;
/** Host name used in RouterAddress definitions */
public final static String PROP_HOST = "host";
public final static String PROP_HOST = RouterAddress.PROP_HOST;
public static final int DEFAULT_COST = 10;
public NTCPAddress(String host, int port) {
@@ -59,23 +59,8 @@ public class NTCPAddress {
_port = -1;
return;
}
String host = addr.getOption(PROP_HOST);
int iport = -1;
if (host == null) {
_host = null;
} else {
_host = host.trim();
String port = addr.getOption(PROP_PORT);
if ( (port != null) && (port.trim().length() > 0) && !("null".equals(port)) ) {
try {
iport = Integer.parseInt(port.trim());
} catch (NumberFormatException nfe) {
Log log = I2PAppContext.getGlobalContext().logManager().getLog(NTCPAddress.class);
log.error("Invalid port [" + port + "]", nfe);
}
}
}
_port = iport;
_host = addr.getOption(PROP_HOST);
_port = addr.getPort();
}
public RouterAddress toRouterAddress() {
@@ -85,7 +70,7 @@ public class NTCPAddress {
RouterAddress addr = new RouterAddress();
addr.setCost(DEFAULT_COST);
addr.setExpiration(null);
//addr.setExpiration(null);
Properties props = new Properties();
props.setProperty(PROP_HOST, _host);
@@ -106,24 +91,11 @@ public class NTCPAddress {
public boolean isPubliclyRoutable() {
return isPubliclyRoutable(_host);
}
public static boolean isPubliclyRoutable(String host) {
if (host == null) return false;
try {
InetAddress addr = InetAddress.getByName(host);
byte quad[] = addr.getAddress();
// allow ipv6 for ntcpaddress, since we've still got ssu
//if (quad.length != 4) {
// if (_log.shouldLog(Log.ERROR))
// _log.error("Refusing IPv6 address (" + host + " / " + addr.getHostAddress() + ") "
// + " since not all peers support it, and we don't support restricted routes");
// return false;
//}
return TransportImpl.isPubliclyRoutable(quad);
} catch (Throwable t) {
//if (_log.shouldLog(Log.WARN))
// _log.warn("Error checking routability", t);
return false;
}
byte quad[] = Addresses.getIP(host);
return TransportImpl.isPubliclyRoutable(quad);
}
@Override

6
router/java/src/net/i2p/router/transport/ntcp/NTCPTransport.java
View File

@@ -298,8 +298,8 @@ public class NTCPTransport extends TransportImpl {
_log.debug("no bid when trying to send to " + peer.toBase64() + " as they don't have an ntcp address");
return null;
}
NTCPAddress naddr = new NTCPAddress(addr);
if ( (naddr.getPort() <= 0) || (naddr.getHost() == null) ) {
byte[] ip = addr.getIP();
if ( (addr.getPort() <= 0) || (ip == null) ) {
_context.statManager().addRateData("ntcp.connectFailedInvalidPort", 1);
markUnreachable(peer);
//_context.shitlist().shitlistRouter(toAddress.getIdentity().calculateHash(), "Invalid NTCP address", STYLE);
@@ -307,7 +307,7 @@ public class NTCPTransport extends TransportImpl {
_log.debug("no bid when trying to send to " + peer.toBase64() + " as they don't have a valid ntcp address");
return null;
}
if (!naddr.isPubliclyRoutable()) {
if (!isPubliclyRoutable(ip)) {
if (! _context.getProperty("i2np.ntcp.allowLocal", "false").equals("true")) {
_context.statManager().addRateData("ntcp.bidRejectedLocalAddress", 1);
markUnreachable(peer);

9
router/java/src/net/i2p/router/transport/udp/PeerTestManager.java
View File

@@ -9,6 +9,7 @@ import java.util.concurrent.LinkedBlockingQueue;
import net.i2p.data.Base64;
import net.i2p.data.DataHelper;
import net.i2p.data.RouterAddress;
import net.i2p.data.RouterInfo;
import net.i2p.data.SessionKey;
import net.i2p.router.CommSystemFacade;
@@ -584,7 +585,13 @@ class PeerTestManager {
aliceIntroKey = new SessionKey(new byte[SessionKey.KEYSIZE_BYTES]);
testInfo.readIntroKey(aliceIntroKey.getData(), 0);
UDPAddress addr = new UDPAddress(charlieInfo.getTargetAddress(UDPTransport.STYLE));
RouterAddress raddr = charlieInfo.getTargetAddress(UDPTransport.STYLE);
if (raddr == null) {
if (_log.shouldLog(Log.WARN))
_log.warn("Unable to pick a charlie");
return;
}
UDPAddress addr = new UDPAddress(raddr);
SessionKey charlieIntroKey = new SessionKey(addr.getIntroKey());
//UDPPacket packet = _packetBuilder.buildPeerTestToAlice(aliceIP, from.getPort(), aliceIntroKey, charlieIntroKey, nonce);

25
router/java/src/net/i2p/router/transport/udp/UDPAddress.java
View File

@@ -12,9 +12,9 @@ import net.i2p.data.SessionKey;
* FIXME public for ConfigNetHelper
*/
public class UDPAddress {
private String _host;
private final String _host;
private InetAddress _hostAddress;
private int _port;
private final int _port;
private byte[] _introKey;
private String _introHosts[];
private InetAddress _introAddresses[];
@@ -23,8 +23,8 @@ public class UDPAddress {
private long _introTags[];
private int _mtu;
public static final String PROP_PORT = "port";
public static final String PROP_HOST = "host";
public static final String PROP_PORT = RouterAddress.PROP_PORT;
public static final String PROP_HOST = RouterAddress.PROP_HOST;
public static final String PROP_INTRO_KEY = "key";
public static final String PROP_MTU = "mtu";
@@ -40,16 +40,13 @@ public class UDPAddress {
public UDPAddress(RouterAddress addr) {
// TODO make everything final
if (addr == null) return;
_host = addr.getOption(PROP_HOST);
if (_host != null) _host = _host.trim();
try {
String port = addr.getOption(PROP_PORT);
if (port != null)
_port = Integer.parseInt(port);
} catch (NumberFormatException nfe) {
_port = -1;
if (addr == null) {
_host = null;
_port = 0;
return;
}
_host = addr.getOption(PROP_HOST);
_port = addr.getPort();
try {
String mtu = addr.getOption(PROP_MTU);
if (mtu != null)
@@ -146,7 +143,7 @@ public class UDPAddress {
}
/**
* @return 0 if unset; -1 if invalid
* @return 0 if unset or invalid
*/
public int getPort() { return _port; }

28
router/java/src/net/i2p/router/transport/udp/UDPTransport.java
View File

@@ -1203,27 +1203,14 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
//UDPAddress ua = new UDPAddress(addr);
//if (ua.getIntroducerCount() <= 0) {
if (addr.getOption("ihost0") == null) {
String host = addr.getOption(UDPAddress.PROP_HOST);
String port = addr.getOption(UDPAddress.PROP_PORT);
if (host == null || port == null) {
byte[] ip = addr.getIP();
int port = addr.getPort();
if (ip == null || port <= 0 ||
(!isValid(ip)) ||
Arrays.equals(ip, getExternalIP())) {
markUnreachable(to);
return null;
}
try {
InetAddress ia = InetAddress.getByName(host);
int iport = Integer.parseInt(port);
if (iport <= 0 || iport > 65535 || (!isValid(ia.getAddress())) ||
Arrays.equals(ia.getAddress(), getExternalIP())) {
markUnreachable(to);
return null;
}
} catch (UnknownHostException uhe) {
markUnreachable(to);
return null;
} catch (NumberFormatException nfe) {
markUnreachable(to);
return null;
}
}
if (!allowConnection())
return _cachedBid[TRANSIENT_FAIL_BID];
@@ -1338,6 +1325,9 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
_fragments.add(msg);
}
/**
* "injected" message from the EstablishmentManager
*/
void send(I2NPMessage msg, PeerState peer) {
if (_log.shouldLog(Log.DEBUG))
_log.debug("Injecting a data message to a new peer: " + peer);
@@ -1447,7 +1437,7 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
addr.setCost(DEFAULT_COST + 1);
else
addr.setCost(DEFAULT_COST);
addr.setExpiration(null);
//addr.setExpiration(null);
addr.setTransportStyle(STYLE);
addr.setOptions(options);