From 5db764de5fa1bddd92518c5727ff75517e1aa416 Mon Sep 17 00:00:00 2001 From: zzz Date: Mon, 5 Jan 2015 12:38:38 +0000 Subject: [PATCH] Blocklist: - Rewrite to read and merge multiple files - Include in update, use version in base dir too - Increase limits - Bug fixes --- build.xml | 1 + installer/resources/blocklist.txt | 16 +- router/java/src/net/i2p/router/Blocklist.java | 242 ++++++++++++------ .../net/i2p/router/startup/WorkingDir.java | 3 +- 4 files changed, 185 insertions(+), 77 deletions(-) diff --git a/build.xml b/build.xml index 4e765e0d6..c26e1ee48 100644 --- a/build.xml +++ b/build.xml @@ -1384,6 +1384,7 @@ ---------------- EARLIER HISTORY IS AVAILABLE IN THE SOURCE PACKAGE" + diff --git a/installer/resources/blocklist.txt b/installer/resources/blocklist.txt index 2262b015b..6b9948762 100644 --- a/installer/resources/blocklist.txt +++ b/installer/resources/blocklist.txt @@ -5,9 +5,18 @@ # When running as a Linux daemon, the configuration directory is /var/lib/i2p # and the install directory is /usr/share/i2p . # -# Blocking is now enabled by default. -# To disable blocking, set router.blocklist.enable=false on configadvanced.jsp, -# or simply delete this file or remove all the entries below, and restart. +# The file in the install directory will be overwritten when you update I2P. +# As of release 0.9.18, the router reads and merges the files in the install directory +# and the config directory. Additionally, if the advanded configuration +# router.blocklist.file=/path/to/otherlist.txt is set, this file will be +# read in and merged as well. +# +# If there is no blocklist.txt file in the configuration directory, create it and +# add entries as desired. +# +# +# Blocking is enabled by default. +# To disable blocking, set router.blocklist.enable=false on configadvanced.jsp. # # Add additional entries as desired, sorting not required. # This file is only read at router startup. @@ -16,6 +25,7 @@ # Please do not block too broadly, it will segment and harm the network. # For example, http://www.bluetack.co.uk/config/splist.zip is very broad and includes Tor users, it is not recommended. # A more reasonable list: http://www.bluetack.co.uk/config/level1.zip +# Note: bluetack blocklists now require a subscription. See https://forum.transmissionbt.com/viewtopic.php?f=2&t=15652 # # We have included the bogons from http://www.team-cymru.org/Services/Bogons/http.html , # but you will have to update your blocklist manually if the bogon list changes. diff --git a/router/java/src/net/i2p/router/Blocklist.java b/router/java/src/net/i2p/router/Blocklist.java index a6fe1d209..33a661870 100644 --- a/router/java/src/net/i2p/router/Blocklist.java +++ b/router/java/src/net/i2p/router/Blocklist.java @@ -76,6 +76,7 @@ public class Blocklist { private final Object _lock = new Object(); private Entry _wrapSave; private final Set _inProcess = new HashSet(4); + // temp private Map _peerBlocklist = new HashMap(4); /** @@ -83,8 +84,8 @@ public class Blocklist { * Note that it's impossible to prevent clogging up * the tables by a determined attacker, esp. on IPv6 */ - private static final int MAX_IPV4_SINGLES = 256; - private static final int MAX_IPV6_SINGLES = 512; + private static final int MAX_IPV4_SINGLES = 8192; + private static final int MAX_IPV6_SINGLES = 4096; private final Set _singleIPBlocklist = new ConcurrentHashSet(4); private final Map _singleIPv6Blocklist = new LHMCache(MAX_IPV6_SINGLES); @@ -102,39 +103,95 @@ public class Blocklist { _log = new Log(Blocklist.class); } - static final String PROP_BLOCKLIST_ENABLED = "router.blocklist.enable"; - static final String PROP_BLOCKLIST_DETAIL = "router.blocklist.detail"; - static final String PROP_BLOCKLIST_FILE = "router.blocklist.file"; - static final String BLOCKLIST_FILE_DEFAULT = "blocklist.txt"; + private static final String PROP_BLOCKLIST_ENABLED = "router.blocklist.enable"; + private static final String PROP_BLOCKLIST_DETAIL = "router.blocklist.detail"; + private static final String PROP_BLOCKLIST_FILE = "router.blocklist.file"; + private static final String BLOCKLIST_FILE_DEFAULT = "blocklist.txt"; + /** + * Loads the following files in-order: + * $I2P/blocklist.txt + * ~/.i2p/blocklist.txt + * File if specified with router.blocklist.file + */ public void startup() { if (! _context.getBooleanPropertyDefaultTrue(PROP_BLOCKLIST_ENABLED)) return; - String file = _context.getProperty(PROP_BLOCKLIST_FILE, BLOCKLIST_FILE_DEFAULT); - // Maybe someday we'll read in multiple files and merge them - // StringTokenizer tok = new StringTokenizer(file, " ,\r\n"); - // while (tok.hasMoreTokens()) - // readBlocklistFile(tok.nextToken()); - Job job = new ReadinJob(file); + List files = new ArrayList(3); + + // install dir + File blFile = new File(_context.getBaseDir(), BLOCKLIST_FILE_DEFAULT); + files.add(blFile); + // config dir + if (!_context.getConfigDir().equals(_context.getBaseDir())) { + blFile = new File(_context.getConfigDir(), BLOCKLIST_FILE_DEFAULT); + files.add(blFile); + } + // user specified + String file = _context.getProperty(PROP_BLOCKLIST_FILE); + if (file != null && !file.equals(BLOCKLIST_FILE_DEFAULT)) { + blFile = new File(file); + if (!blFile.isAbsolute()) + blFile = new File(_context.getConfigDir(), file); + files.add(blFile); + } + Job job = new ReadinJob(files); job.getTiming().setStartAfter(_context.clock().now() + 2*60*1000); _context.jobQueue().addJob(job); } private class ReadinJob extends JobImpl { - private final String _file; - public ReadinJob (String f) { + private final List _files; + + /** + * @param files not necessarily existing, but avoid dups + */ + public ReadinJob (List files) { super(_context); - _file = f; + _files = files; } + public String getName() { return "Read Blocklist"; } + public void runJob() { + allocate(_files); + if (_blocklist == null) + return; + int ccount = process(); + if (_blocklist == null) + return; + if (ccount <= 0) { + disable(); + return; + } + merge(ccount); + if (_log.shouldLog(Log.WARN)) { + if (_blocklistSize <= 0) + return; + FloodfillNetworkDatabaseFacade fndf = (FloodfillNetworkDatabaseFacade) _context.netDb(); + int count = 0; + for (RouterInfo ri : fndf.getKnownRouterData()) { + Hash peer = ri.getIdentity().getHash(); + if (isBlocklisted(peer)) + count++; + } + if (count > 0) + _log.warn("Blocklisted " + count + " routers in the netDb"); + } + _peerBlocklist = null; + } + + private int process() { + int count = 0; synchronized (_lock) { try { - readBlocklistFile(_file); + for (File f : _files) { + count = readBlocklistFile(f, count); + } } catch (OutOfMemoryError oom) { _log.log(Log.CRIT, "OOM processing the blocklist"); disable(); - return; + return 0; } } for (Hash peer : _peerBlocklist.keySet()) { @@ -146,20 +203,8 @@ public class Blocklist { reason = _x("Banned by router hash"); _context.banlist().banlistRouterForever(peer, reason, comment); } - _peerBlocklist = null; - - if (_blocklistSize <= 0) - return; - FloodfillNetworkDatabaseFacade fndf = (FloodfillNetworkDatabaseFacade) _context.netDb(); - int count = 0; - for (Iterator iter = fndf.getKnownRouterData().iterator(); iter.hasNext(); ) { - RouterInfo ri = iter.next(); - Hash peer = ri.getIdentity().getHash(); - if (isBlocklisted(peer)) - count++; - } - if (count > 0 && _log.shouldLog(Log.WARN)) - _log.warn("Blocklisted " + count + " routers in the netDb."); + _peerBlocklist.clear(); + return count; } } @@ -171,6 +216,23 @@ public class Blocklist { } } + /** + * @return success + * @since 0.9.18 split out from readBlocklistFile() + */ + private void allocate(List files) { + int maxSize = 0; + for (File f : files) { + maxSize += getSize(f); + } + try { + _blocklist = new long[maxSize + files.size()]; // extra for wrapsave + } catch (OutOfMemoryError oom) { + _log.log(Log.CRIT, "OOM creating the blocklist"); + disable(); + } + } + /** * Read in and parse the blocklist. * The blocklist need not be sorted, and may contain overlapping entries. @@ -192,34 +254,31 @@ public class Blocklist { * http://www.bluetack.co.uk/forums/index.php?autocom=faq&CODE=02&qid=17 * http://blocklist.googlepages.com/ * http://www.cymru.com/Documents/bogon-list.html + * + * + * Must call allocate() before and merge() after. + * + * @param count current number of entries + * @return new number of entries */ - private void readBlocklistFile(String file) { - File BLFile = new File(file); - if (!BLFile.isAbsolute()) - BLFile = new File(_context.getConfigDir(), file); - if (BLFile == null || (!BLFile.exists()) || BLFile.length() <= 0) { + private int readBlocklistFile(File blFile, int count) { + if (blFile == null || (!blFile.exists()) || blFile.length() <= 0) { if (_log.shouldLog(Log.WARN)) - _log.warn("Blocklist file not found: " + file); - return; + _log.warn("Blocklist file not found: " + blFile); + return count; } + long start = _context.clock().now(); - int maxSize = getSize(BLFile); - try { - _blocklist = new long[maxSize + 1]; // extra for wrapsave - } catch (OutOfMemoryError oom) { - _log.log(Log.CRIT, "OOM creating the blocklist"); - return; - } - int count = 0; + int oldcount = count; int badcount = 0; int peercount = 0; long ipcount = 0; BufferedReader br = null; try { br = new BufferedReader(new InputStreamReader( - new FileInputStream(BLFile), "UTF-8")); + new FileInputStream(blFile), "UTF-8")); String buf = null; - while ((buf = br.readLine()) != null && count < maxSize) { + while ((buf = br.readLine()) != null) { Entry e = parse(buf, true); if (e == null) { badcount++; @@ -237,23 +296,40 @@ public class Blocklist { ipcount += 1 + toInt(ip2) - toInt(ip1); // includes dups, oh well } } catch (IOException ioe) { - _blocklist = null; if (_log.shouldLog(Log.ERROR)) - _log.error("Error reading the BLFile", ioe); - return; + _log.error("Error reading the blocklist file", ioe); + return count; } catch (OutOfMemoryError oom) { _blocklist = null; _log.log(Log.CRIT, "OOM reading the blocklist"); - return; + return count; } finally { if (br != null) try { br.close(); } catch (IOException ioe) {} } if (_wrapSave != null) { + // the extra record generated in parse() by a line that + // wrapped around 128.0.0.0 store(_wrapSave.ip1, _wrapSave.ip2, count++); ipcount += 1 + toInt(_wrapSave.ip2) - toInt(_wrapSave.ip1); + _wrapSave = null; } + if (_log.shouldLog(Log.INFO)) { + _log.info("Stats for " + blFile); + _log.info("Removed " + badcount + " bad entries and comment lines"); + _log.info("Read " + (count - oldcount) + " valid entries from the blocklist " + blFile); + _log.info("Blocking " + ipcount + " IPs and " + peercount + " hashes"); + _log.info("Blocklist processing finished, time: " + (_context.clock().now() - start)); + } + return count; + } + /** + * @param count valid entries in _blocklist + * @since 0.9.18 split out from readBlocklistFile() + */ + private void merge(int count) { + long start = _context.clock().now(); // This is a standard signed sort, so the entries will be ordered // 128.0.0.0 ... 255.255.255.255 0.0.0.0 .... 127.255.255.255 // But that's ok. @@ -273,15 +349,17 @@ public class Blocklist { } _blocklistSize = count - removed; if (_log.shouldLog(Log.INFO)) { - _log.info("Removed " + badcount + " bad entries and comment lines"); - _log.info("Read " + count + " valid entries from the blocklist " + BLFile); + _log.info("Merged Stats"); + _log.info("Read " + count + " total entries from the blocklists"); _log.info("Merged " + removed + " overlapping entries"); _log.info("Result is " + _blocklistSize + " entries"); - _log.info("Blocking " + ipcount + " IPs and " + peercount + " hashes"); _log.info("Blocklist processing finished, time: " + (_context.clock().now() - start)); } } + /** + * The result of parsing one line + */ private static class Entry { final String comment; final byte ip1[]; @@ -296,6 +374,9 @@ public class Blocklist { } } + /** + * Parse one line, returning a temp data structure with the result + */ private Entry parse(String buf, boolean shouldLog) { byte[] ip1; byte[] ip2; @@ -401,19 +482,21 @@ public class Blocklist { * so we can size our array. * This is i/o inefficient, but memory-efficient, which is what we want. */ - private int getSize(File BLFile) { - if ( (!BLFile.exists()) || (BLFile.length() <= 0) ) return 0; + private int getSize(File blFile) { + if ( (!blFile.exists()) || (blFile.length() <= 0) ) return 0; int lines = 0; BufferedReader br = null; try { br = new BufferedReader(new InputStreamReader( - new FileInputStream(BLFile), "ISO-8859-1")); - while (br.readLine() != null) { - lines++; + new FileInputStream(blFile), "ISO-8859-1")); + String s; + while ((s = br.readLine()) != null) { + if (s.length() > 0 && !s.startsWith("#")) + lines++; } } catch (IOException ioe) { if (_log.shouldLog(Log.WARN)) - _log.warn("Error reading the BLFile", ioe); + _log.warn("Error reading the blocklist file", ioe); return 0; } finally { if (br != null) try { br.close(); } catch (IOException ioe) {} @@ -666,7 +749,7 @@ public class Blocklist { private void store(int ip1, int ip2, int idx) { long entry = ((long) ip1) << 32; - entry |= ip2; + entry |= ((long)ip2) & 0xffffffff; _blocklist[idx] = entry; } @@ -752,16 +835,29 @@ public class Blocklist { * Additional jobs can wait. * Although could this clog up the job queue runners? Yes. * So we also stagger these jobs. - *(Map.Entry) + * */ private synchronized void banlistForever(Hash peer, List ips) { - String file = _context.getProperty(PROP_BLOCKLIST_FILE, BLOCKLIST_FILE_DEFAULT); - File BLFile = new File(file); - if (!BLFile.isAbsolute()) - BLFile = new File(_context.getConfigDir(), file); - if (BLFile == null || (!BLFile.exists()) || BLFile.length() <= 0) { - if (_log.shouldLog(Log.ERROR)) - _log.error("Blocklist file not found: " + file); + // This only checks one file for now, pick the best one + // user specified + File blFile = null; + String file = _context.getProperty(PROP_BLOCKLIST_FILE); + if (file != null) { + blFile = new File(file); + if (!blFile.isAbsolute()) + blFile = new File(_context.getConfigDir(), file); + if (!blFile.exists()) + blFile = null; + } + // install dir + if (blFile == null) + blFile = new File(_context.getBaseDir(), BLOCKLIST_FILE_DEFAULT); + + if ((!blFile.exists()) || blFile.length() <= 0) { + // just ban it and be done + if (_log.shouldLog(Log.WARN)) + _log.warn("Banlisting " + peer); + _context.banlist().banlistRouterForever(peer, "Banned"); return; } @@ -772,7 +868,7 @@ public class Blocklist { BufferedReader br = null; try { br = new BufferedReader(new InputStreamReader( - new FileInputStream(BLFile), "UTF-8")); + new FileInputStream(blFile), "UTF-8")); String buf = null; // assume the file is unsorted, so go through the whole thing while ((buf = br.readLine()) != null) { @@ -798,7 +894,7 @@ public class Blocklist { } } catch (IOException ioe) { if (_log.shouldLog(Log.WARN)) - _log.warn("Error reading the BLFile", ioe); + _log.warn("Error reading the blocklist file", ioe); } finally { if (br != null) try { br.close(); } catch (IOException ioe) {} } @@ -867,7 +963,7 @@ public class Blocklist { int max = Math.min(_blocklistSize, MAX_DISPLAY); int displayed = 0; // first 0 - 127 - for (int i = 0; i < max; i++) { + for (int i = 0; i < _blocklistSize && displayed < max; i++) { int from = getFrom(_blocklist[i]); if (from < 0) continue; diff --git a/router/java/src/net/i2p/router/startup/WorkingDir.java b/router/java/src/net/i2p/router/startup/WorkingDir.java index 23149c759..3be6b6309 100644 --- a/router/java/src/net/i2p/router/startup/WorkingDir.java +++ b/router/java/src/net/i2p/router/startup/WorkingDir.java @@ -276,7 +276,8 @@ public class WorkingDir { // base install - files // We don't currently have a default router.config, logger.config, susimail.config, or webapps.config in the base distribution, // but distros might put one in - "blocklist.txt,hosts.txt,i2psnark.config,i2ptunnel.config,jetty-i2psnark.xml," + + // blocklist.txt now accessed in base dir, user can add another in config dir if desired + "hosts.txt,i2psnark.config,i2ptunnel.config,jetty-i2psnark.xml," + "logger.config,router.config,susimail.config,systray.config,webapps.config"; private static boolean migrate(String list, File olddir, File todir) {