From 5d06de860848c3d564bb8c270b2c011341bb2d23 Mon Sep 17 00:00:00 2001 From: zzz Date: Fri, 14 Dec 2018 16:54:10 +0000 Subject: [PATCH] Sybil: Class for persisting results, related refactoring --- .../net/i2p/router/sybil/PersistSybil.java | 216 ++++++++++++++++++ .../java/src/net/i2p/router/sybil/Points.java | 102 +++++++++ .../src/net/i2p/router/sybil/package.html | 9 + .../i2p/router/web/helpers/SybilRenderer.java | 38 +-- build.xml | 2 +- 5 files changed, 340 insertions(+), 27 deletions(-) create mode 100644 apps/routerconsole/java/src/net/i2p/router/sybil/PersistSybil.java create mode 100644 apps/routerconsole/java/src/net/i2p/router/sybil/Points.java create mode 100644 apps/routerconsole/java/src/net/i2p/router/sybil/package.html diff --git a/apps/routerconsole/java/src/net/i2p/router/sybil/PersistSybil.java b/apps/routerconsole/java/src/net/i2p/router/sybil/PersistSybil.java new file mode 100644 index 000000000..cc9523d92 --- /dev/null +++ b/apps/routerconsole/java/src/net/i2p/router/sybil/PersistSybil.java @@ -0,0 +1,216 @@ +package net.i2p.router.sybil; + +import java.io.BufferedReader; +import java.util.Collections; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Reader; +import java.io.Writer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +import net.i2p.I2PAppContext; +import net.i2p.data.Base64; +import net.i2p.data.DataFormatException; +import net.i2p.data.DataHelper; +import net.i2p.data.Hash; +import net.i2p.router.web.helpers.SybilRenderer; +import net.i2p.util.Log; +import net.i2p.util.FileSuffixFilter; +import net.i2p.util.SecureDirectory; +import net.i2p.util.SecureFileOutputStream; + +/** + * Store and retrieve analysis files from disk. + * Each file is named with a timestamp. + * + * @since 0.9.38 + */ +public class PersistSybil { + + private final I2PAppContext _context; + private final Log _log; + + private static final String DIR = "sybil-analysis/results"; + private static final String PFX = "sybil-"; + private static final String SFX = ".txt.gz"; + + public PersistSybil(I2PAppContext ctx) { + _context = ctx; + _log = ctx.logManager().getLog(PersistSybil.class); + } + + /** + * Store each entry. + * + * @param entries each one should be "entry" at the root + */ + public synchronized void store(long date, Map entries) throws IOException { + File dir = new SecureDirectory(_context.getConfigDir(), DIR); + if (!dir.exists()) + dir.mkdirs(); + File file = new File(dir, PFX + date + SFX); + StringBuilder buf = new StringBuilder(128); + Writer out = null; + try { + out = new OutputStreamWriter(new GZIPOutputStream(new SecureFileOutputStream(file))); + for (Map.Entry entry : entries.entrySet()) { + Hash h = entry.getKey(); + Points p = entry.getValue(); + buf.append(h.toBase64()).append(':'); + p.toString(buf); + buf.append('\n'); + out.write(buf.toString()); + buf.setLength(0); + } + } finally { + if (out != null) try { out.close(); } catch (IOException ioe) {} + } + } + + /** + * The list of stored analysis sets, as a time stamp. + * + * @return non-null, sorted by updated date, newest first + */ + public synchronized List load() { + File dir = new File(_context.getConfigDir(), DIR); + List rv = new ArrayList(); + File[] files = dir.listFiles(new FileSuffixFilter(PFX, SFX)); + if (files == null) + return rv; + for (File file : files) { + try { + String name = file.getName(); + long d = Long.parseLong(name.substring(PFX.length(), name.length() - SFX.length())); + rv.add(Long.valueOf(d)); + } catch (NumberFormatException nfe) {} + } + Collections.sort(rv); + return rv; + } + + /** + * Load the analysis for a certain date. + * + * @return non-null, unsorted + */ + public synchronized Map load(long date) throws IOException { + File dir = new File(_context.getConfigDir(), DIR); + File file = new File(dir, PFX + date + SFX); + Map rv = new HashMap(); + BufferedReader in = null; + try { + in = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(file)))); + String line; + while ((line = in.readLine()) != null) { + int colon = line.indexOf(':'); + if (colon != 44) + continue; + if (line.length() < 46) + continue; + Hash h = new Hash(); + try { + h.fromBase64(line.substring(0, 44)); + } catch (DataFormatException dfe) { + continue; + } + Points p = Points.fromString(line.substring(45)); + if (p == null) + continue; + rv.put(h, p); + } + } finally { + if (in != null) try { in.close(); } catch (IOException ioe) {} + } + return rv; + } + + /** + * Load all the analysis for a certain hash. + * + * @return non-null, unsorted + */ + public synchronized Map load(Hash h) throws IOException { + String bh = h.toBase64() + ':'; + File dir = new File(_context.getConfigDir(), DIR); + Map rv = new HashMap(); + List dates = load(); + for (Long date : dates) { + File file = new File(dir, PFX + date + SFX); + BufferedReader in = null; + try { + in = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(file)))); + String line; + while ((line = in.readLine()) != null) { + if (!line.startsWith(bh)) + continue; + if (line.length() < 46) + continue; + Points p = Points.fromString(line.substring(45)); + if (p == null) + continue; + rv.put(date, p); + } + } finally { + if (in != null) try { in.close(); } catch (IOException ioe) {} + } + } + return rv; + } + + /** + * Delete the file for a particular date + * + * @return success + */ + public synchronized boolean delete(long date) { + File dir = new File(_context.getConfigDir(), DIR); + File file = new File(dir, PFX + date + SFX); + return file.delete(); + } + +/**** + public static void main(String[] args) { + I2PAppContext ctx = I2PAppContext.getGlobalContext(); + PersistSybil ps = new PersistSybil(ctx); + byte[] b = new byte[32]; + ctx.random().nextBytes(b); + Hash h = new Hash(b); + String rsn = "Test reason"; + Points p = new Points(1.234, rsn); + rsn = "Test reason2"; + p.addPoints(2.345, rsn); + Map map = new HashMap(); + map.put(h, p); + b = new byte[32]; + ctx.random().nextBytes(b); + h = new Hash(b); + map.put(h, p); + try { + long now = System.currentTimeMillis(); + System.out.println("storing entries: " + map.size()); + ps.store(System.currentTimeMillis(), map); + List dates = ps.load(); + System.out.println("Found sets: " + dates.size()); + map = ps.load(Long.valueOf(now)); + System.out.println("loaded entries: " + map.size()); + for (Map.Entry e : map.entrySet()) { + System.out.println(e.getKey().toString() + ": " + e.getValue()); + } + } catch (IOException ioe) { + System.out.println("store error from " + args[0]); + ioe.printStackTrace(); + } + } +****/ +} diff --git a/apps/routerconsole/java/src/net/i2p/router/sybil/Points.java b/apps/routerconsole/java/src/net/i2p/router/sybil/Points.java new file mode 100644 index 000000000..7386fe2e0 --- /dev/null +++ b/apps/routerconsole/java/src/net/i2p/router/sybil/Points.java @@ -0,0 +1,102 @@ +package net.i2p.router.sybil; + +import java.text.DecimalFormat; +import java.util.ArrayList; +import java.util.List; + +import net.i2p.data.DataHelper; + +/** + * A total score and a List of reason Strings + * + * @since 0.9.38 moved from SybilRenderer + */ +public class Points implements Comparable { + private double points; + private final List reasons; + + /** + * @since 0.9.38 + */ + private Points() { + reasons = new ArrayList(4); + } + + public Points(double d, String reason) { + this(); + addPoints(d, reason); + } + + /** + * @since 0.9.38 + */ + public double getPoints() { + return points; + } + + /** + * @since 0.9.38 + */ + public List getReasons() { + return reasons; + } + + /** + * @since 0.9.38 + */ + public void addPoints(double d, String reason) { + points += d; + DecimalFormat format = new DecimalFormat("#0.00"); + String rsn = format.format(d) + ": " + reason; + reasons.add(rsn); + } + + public int compareTo(Points r) { + return Double.compare(points, r.points); + } + + /** + * @since 0.9.38 + */ + @Override + public String toString() { + StringBuilder buf = new StringBuilder(128); + toString(buf); + return buf.toString(); + } + + /** + * For persistence. + * Total points and reasons, comma separated, no newline + * @since 0.9.38 + */ + public void toString(StringBuilder buf) { + buf.append(points); + for (String r : reasons) { + buf.append(',').append(r); + } + } + + /** + * For persistence. + * @return null on failure + * @since 0.9.38 + */ + public static Points fromString(String s) { + String[] ss = DataHelper.split(s, ","); + if (ss.length < 2) + return null; + double d; + try { + d = Double.parseDouble(ss[0]); + } catch (NumberFormatException nfe) { + return null; + } + Points rv = new Points(d, ss[1]); + for (int i = 2; i < ss.length; i++) { + rv.reasons.add(ss[i]); + } + return rv; + } +} + diff --git a/apps/routerconsole/java/src/net/i2p/router/sybil/package.html b/apps/routerconsole/java/src/net/i2p/router/sybil/package.html new file mode 100644 index 000000000..a1219ceb5 --- /dev/null +++ b/apps/routerconsole/java/src/net/i2p/router/sybil/package.html @@ -0,0 +1,9 @@ + +

+Classes to run offline Sybil analysis, and to +store and load the results. +

+

+Since 0.9.38. +

+ diff --git a/apps/routerconsole/java/src/net/i2p/router/web/helpers/SybilRenderer.java b/apps/routerconsole/java/src/net/i2p/router/web/helpers/SybilRenderer.java index 73ab56c61..7dab4226d 100644 --- a/apps/routerconsole/java/src/net/i2p/router/web/helpers/SybilRenderer.java +++ b/apps/routerconsole/java/src/net/i2p/router/web/helpers/SybilRenderer.java @@ -30,6 +30,7 @@ import net.i2p.router.TunnelPoolSettings; import net.i2p.router.crypto.FamilyKeyCrypto; import net.i2p.router.peermanager.DBHistory; import net.i2p.router.peermanager.PeerProfile; +import net.i2p.router.sybil.Points; import net.i2p.router.tunnel.pool.TunnelPool; import net.i2p.router.util.HashDistance; import net.i2p.router.web.Messages; @@ -50,7 +51,7 @@ import net.i2p.util.VersionComparator; * @since 0.9.24 * */ -class SybilRenderer { +public class SybilRenderer { private final RouterContext _context; private final DecimalFormat fmt = new DecimalFormat("#0.00"); @@ -102,23 +103,6 @@ class SybilRenderer { } } - /** - * A total score and a List of reason Strings - */ - public static class Points implements Comparable { - private double points; - private final List reasons; - - public Points(double points, String reason) { - this.points = points; - reasons = new ArrayList(4); - reasons.add(reason); - } - public int compareTo(Points r) { - return Double.compare(points, r.points); - } - } - private static class PointsComparator implements Comparator, Serializable { private final Map _points; @@ -159,6 +143,7 @@ class SybilRenderer { * Merge points1 into points2. * points1 is unmodified. */ +/**** private void mergePoints(Map points1, Map points2) { for (Map.Entry e : points1.entrySet()) { Hash h = e.getKey(); @@ -172,15 +157,15 @@ class SybilRenderer { } } } +****/ + /** */ private void addPoints(Map points, Hash h, double d, String reason) { - String rsn = fmt.format(d) + ": " + reason; Points dd = points.get(h); if (dd != null) { - dd.points += d; - dd.reasons.add(rsn); + dd.addPoints(d, reason); } else { - points.put(h, new Points(d, rsn)); + points.put(h, new Points(d, reason)); } } @@ -323,13 +308,14 @@ class SybilRenderer { if (ri == null) continue; Points pp = points.get(h); - double p = pp.points; + double p = pp.getPoints(); if (p < MIN_DISPLAY_POINTS) break; // sorted buf.append("

Threat Points: " + fmt.format(p) + "

    "); - if (pp.reasons.size() > 1) - Collections.sort(pp.reasons, new ReasonComparator()); - for (String s : pp.reasons) { + List reasons = pp.getReasons(); + if (reasons.size() > 1) + Collections.sort(reasons, new ReasonComparator()); + for (String s : reasons) { int c = s.indexOf(':'); if (c <= 0) continue; diff --git a/build.xml b/build.xml index e836af31c..ef9066fc9 100644 --- a/build.xml +++ b/build.xml @@ -825,7 +825,7 @@ - +