From e466331407b6cf29fffc5c4188b0e90b2b9946fa Mon Sep 17 00:00:00 2001 From: zzz Date: Wed, 9 Sep 2020 15:26:24 +0000 Subject: [PATCH] GeoIP: Add methods to get all IPs for a country from a geoip 1 or 2 database, to prep for a country blocklist (ticket #2759) WIP - not hooked in yet --- router/java/src/com/maxmind/db/Reader.java | 124 ++++++++++++++++++ .../src/com/maxmind/geoip/LookupService.java | 86 ++++++++++++ .../com/maxmind/geoip2/DatabaseReader.java | 14 ++ 3 files changed, 224 insertions(+) diff --git a/router/java/src/com/maxmind/db/Reader.java b/router/java/src/com/maxmind/db/Reader.java index 65a705097..0f77850c7 100644 --- a/router/java/src/com/maxmind/db/Reader.java +++ b/router/java/src/com/maxmind/db/Reader.java @@ -4,9 +4,12 @@ import java.io.Closeable; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.Writer; import java.net.InetAddress; import java.nio.ByteBuffer; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicReference; /** @@ -149,6 +152,127 @@ public final class Reader implements Closeable { return this.resolveDataPointer(buffer, pointer); } + /** + * I2P - + * Write all IPv4 address ranges for the given country to out. + * + * @param country two-letter uppper-case + * @param out caller must close + * @since 0.9.48 + */ + public void countryToIP(String country, Writer out) throws IOException { + Walker walker = new Walker(country, out); + walker.walk(); + } + + /** + * I2P + * @since 0.9.48 + */ + private class Walker { + private final String _country; + private final Writer _out; + private final int _nodeCount; + private final ByteBuffer _buffer; + private final Set _negativeCache; + //private boolean _ipv6; + private int _countryRecord = -1; + + /** + * Write all IPv4 address ranges for the given country to out. + * + * @param country two-letter uppper-case + */ + public Walker(String country, Writer out) throws IOException { + _country = country; + _out = out; + _nodeCount = metadata.getNodeCount(); + _buffer = getBufferHolder().get(); + _negativeCache = new HashSet(256); + } + + /** only call once */ + public void walk() throws IOException { + _out.write("# IPs for country " + _country + " from GeoIP2 database\n"); + int record = startNode(32); + walk(record, 0, 0); + // TODO if supported in blocklist + //record = startNode(128); + //_ipv6 = true; + //walk(record, 0, 0); + } + + /** + * Recursive, depth first + * @param ip big endian + */ + private void walk(int record, int ip, int depth) throws IOException { + if (record == _nodeCount) { + return; + } + if (record > _nodeCount) { + boolean found; + if (_countryRecord < 0) { + // Without a negative cache, perversely, the rarest + // countries take the longest, because it takes longer to + // find the right record and set _countryRecord. + // The negative cache speeds up the search for an unknown country by 25x. + // If we could find the country record in advance that would + // be even better, but there's no way to do that other than + // "priming" it with a known IP. + if (_negativeCache.contains(Integer.valueOf(record))) + return; + // This is the slow part + // we only need to read and parse the country record once + // wouldn't work on a city database? + Object o = resolveDataPointer(_buffer, record); + if (!(o instanceof Map)) + return; + Map m = (Map) o; + o = m.get("country"); + if (!(o instanceof Map)) + return; + m = (Map) o; + o = m.get("iso_code"); + found = _country.equals(o); + if (found) { + _countryRecord = record; + _negativeCache.clear(); + } else { + if (_negativeCache.size() < 10000) // don't blow up on city database? + _negativeCache.add(Integer.valueOf(record)); + } + } else { + found = record == _countryRecord; + } + if (found) { + String sip; + //if (_ipv6) { + // sip = Integer.toHexString((ip >> 16) & 0xffff) + ":" + + // Integer.toHexString(ip & 0xffff) + "::/" + depth; + //} else { + sip = ((ip >> 24) & 0xff) + "." + + ((ip >> 16) & 0xff) + '.' + + ((ip >> 8) & 0xff) + '.' + + (ip & 0xff); + //} + _out.write(sip); + if (depth < 32) { + _out.write('/'); + _out.write(Integer.toString(depth)); + } + _out.write('\n'); + } + return; + } + if (depth >= 32) + return; + walk(readNode(_buffer, record, 0), ip, depth + 1); + ip |= 1 << (31 - depth); + walk(readNode(_buffer, record, 1), ip, depth + 1); + } + } + private BufferHolder getBufferHolder() throws ClosedDatabaseException { BufferHolder bufferHolder = this.bufferHolderReference.get(); if (bufferHolder == null) { diff --git a/router/java/src/com/maxmind/geoip/LookupService.java b/router/java/src/com/maxmind/geoip/LookupService.java index 94687fb55..9f122aef8 100644 --- a/router/java/src/com/maxmind/geoip/LookupService.java +++ b/router/java/src/com/maxmind/geoip/LookupService.java @@ -23,6 +23,7 @@ package com.maxmind.geoip; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; +import java.io.Writer; import java.net.InetAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; @@ -31,6 +32,7 @@ import java.nio.charset.Charset; import java.nio.charset.CharsetDecoder; import java.util.Arrays; import java.util.List; +import java.util.Locale; /** * Provides a lookup service for information based on an IP address. The @@ -501,6 +503,90 @@ public class LookupService { } } + /** + * I2P - + * Write all IPv4 address ranges for the given country to out. + * + * @param country two-letter case-insensitive + * @param out caller must close + * @since 0.9.48 + */ + public synchronized void countryToIP(String country, Writer out) throws IOException { + if (file == null && (dboptions & GEOIP_MEMORY_CACHE) == 0) { + throw new IllegalStateException("Database has been closed."); + } + country = country.toUpperCase(Locale.US); + // get the country ID we're looking for + int id = 0; + for (int i = 0; i < countryCode.length; i++) { + if (countryCode[i].equals(country)) { + id = i; + break; + } + } + if (id <= 0) + return; + // segment + id += COUNTRY_BEGIN; + Walker walker = new Walker(id, out); + out.write("# IPs for country " + country + " from GeoIP database\n"); + walker.walk(); + } + + /** + * I2P + * @since 0.9.48 + */ + private class Walker { + private final int _country; + private final Writer _out; + private final byte[] _buf = new byte[2 * MAX_RECORD_LENGTH]; + private final int[] _x = new int[2]; + private final int _dbs0 = databaseSegments[0]; + + /** + * @param country the segment + */ + public Walker(int country, Writer out) throws IOException { + _country = country; + _out = out; + } + + /** only call once */ + public void walk() throws IOException { + walk(0, 0, 31); + } + + /** + * Recursive, depth first + * @param ip big endian + */ + private void walk(int offset, int ip, int depth) throws IOException { + if (offset >= _dbs0) { + if (offset == _country) { + String sip = ((ip >> 24) & 0xff) + "." + + ((ip >> 16) & 0xff) + '.' + + ((ip >> 8) & 0xff) + '.' + + (ip & 0xff); + _out.write(sip); + if (depth >= 0) { + _out.write('/'); + _out.write(Integer.toString(31 - depth)); + } + _out.write('\n'); + } + return; + } + if (depth < 0) + return; + readNode(_buf, _x, offset); + int x1 = _x[1]; + walk(_x[0], ip, depth - 1); + ip |= 1 << depth; + walk(x1, ip, depth - 1); + } + } + public int getID(String ipAddress) { InetAddress addr; try { diff --git a/router/java/src/com/maxmind/geoip2/DatabaseReader.java b/router/java/src/com/maxmind/geoip2/DatabaseReader.java index 3c6af9aed..db82acfeb 100644 --- a/router/java/src/com/maxmind/geoip2/DatabaseReader.java +++ b/router/java/src/com/maxmind/geoip2/DatabaseReader.java @@ -7,9 +7,11 @@ import java.io.Closeable; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.Writer; import java.net.InetAddress; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Map; /** @@ -228,6 +230,18 @@ public class DatabaseReader implements Closeable { } + /** + * I2P - + * Write all IPv4 address ranges for the given country to out. + * + * @param country two-letter case-insensitive + * @param out caller must close + * @since 0.9.48 + */ + public void countryToIP(String country, Writer out) throws IOException { + reader.countryToIP(country.toUpperCase(Locale.US), out); + } + /** * @return the metadata for the open MaxMind DB file. */