From cd76457128584a5ee0760ed8b4060e7054524243 Mon Sep 17 00:00:00 2001 From: zzz Date: Fri, 5 May 2017 12:08:49 +0000 Subject: [PATCH] i2psnark: Initial support for ut_comment, no UI yet --- .../src/org/klomp/snark/CompleteListener.java | 13 + .../src/org/klomp/snark/ExtensionHandler.java | 123 +++++- .../src/org/klomp/snark/I2PSnarkUtil.java | 42 +++ .../java/src/org/klomp/snark/Peer.java | 14 +- .../src/org/klomp/snark/PeerCheckerTask.java | 11 +- .../src/org/klomp/snark/PeerCoordinator.java | 67 ++++ .../src/org/klomp/snark/PeerListener.java | 16 + .../java/src/org/klomp/snark/Snark.java | 54 +++ .../src/org/klomp/snark/SnarkManager.java | 115 +++++- .../src/org/klomp/snark/UpdateRunner.java | 13 + .../src/org/klomp/snark/comments/Comment.java | 219 +++++++++++ .../org/klomp/snark/comments/CommentSet.java | 352 ++++++++++++++++++ .../src/org/klomp/snark/comments/package.html | 7 + .../org/klomp/snark/web/I2PSnarkServlet.java | 6 +- history.txt | 4 + .../src/net/i2p/router/RouterVersion.java | 2 +- 16 files changed, 1047 insertions(+), 11 deletions(-) create mode 100644 apps/i2psnark/java/src/org/klomp/snark/comments/Comment.java create mode 100644 apps/i2psnark/java/src/org/klomp/snark/comments/CommentSet.java create mode 100644 apps/i2psnark/java/src/org/klomp/snark/comments/package.html diff --git a/apps/i2psnark/java/src/org/klomp/snark/CompleteListener.java b/apps/i2psnark/java/src/org/klomp/snark/CompleteListener.java index 1d48654ba..353825be8 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/CompleteListener.java +++ b/apps/i2psnark/java/src/org/klomp/snark/CompleteListener.java @@ -21,6 +21,9 @@ package org.klomp.snark; +import org.klomp.snark.comments.CommentSet; + + /** * Callback for Snark events. * @since 0.9.4 moved from Snark.java @@ -65,4 +68,14 @@ public interface CompleteListener { * @since 0.9.15 */ public long getSavedUploaded(Snark snark); + + /** + * @since 0.9.31 + */ + public CommentSet getSavedComments(Snark snark); + + /** + * @since 0.9.31 + */ + public void locked_saveComments(Snark snark, CommentSet comments); } diff --git a/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java b/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java index a711d9a05..357965e7e 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java +++ b/apps/i2psnark/java/src/org/klomp/snark/ExtensionHandler.java @@ -14,6 +14,8 @@ import net.i2p.util.Log; import org.klomp.snark.bencode.BDecoder; import org.klomp.snark.bencode.BEncoder; import org.klomp.snark.bencode.BEValue; +import org.klomp.snark.comments.Comment; +import org.klomp.snark.comments.CommentSet; /** * REF: BEP 10 Extension Protocol @@ -31,6 +33,10 @@ abstract class ExtensionHandler { public static final int ID_DHT = 3; /** not using the option bit since the compact format is different */ public static final String TYPE_DHT = "i2p_dht"; + /** @since 0.9.31 */ + public static final int ID_COMMENT = 4; + /** @since 0.9.31 */ + public static final String TYPE_COMMENT = "ut_comment"; /** Pieces * SHA1 Hash length, + 25% extra for file names, bencoding overhead, etc */ private static final int MAX_METADATA_SIZE = Storage.MAX_PIECES * 20 * 5 / 4; private static final int PARALLEL_REQUESTS = 3; @@ -40,9 +46,10 @@ abstract class ExtensionHandler { * @param metasize -1 if unknown * @param pexAndMetadata advertise these capabilities * @param dht advertise DHT capability + * @param comment advertise ut_comment capability * @return bencoded outgoing handshake message */ - public static byte[] getHandshake(int metasize, boolean pexAndMetadata, boolean dht, boolean uploadOnly) { + public static byte[] getHandshake(int metasize, boolean pexAndMetadata, boolean dht, boolean uploadOnly, boolean comment) { Map handshake = new HashMap(); Map m = new HashMap(); if (pexAndMetadata) { @@ -54,6 +61,9 @@ abstract class ExtensionHandler { if (dht) { m.put(TYPE_DHT, Integer.valueOf(ID_DHT)); } + if (comment) { + m.put(TYPE_COMMENT, Integer.valueOf(ID_COMMENT)); + } // include the map even if empty so the far-end doesn't NPE handshake.put("m", m); handshake.put("p", Integer.valueOf(TrackerClient.PORT)); @@ -77,6 +87,8 @@ abstract class ExtensionHandler { handlePEX(peer, listener, bs, log); else if (id == ID_DHT) handleDHT(peer, listener, bs, log); + else if (id == ID_COMMENT) + handleComment(peer, listener, bs, log); else if (log.shouldLog(Log.INFO)) log.info("Unknown extension msg " + id + " from " + peer); } @@ -430,4 +442,113 @@ abstract class ExtensionHandler { // log.info("DHT msg exception to " + peer, e); } } + + /** + * Handle comment request and response + * + * Ref: https://blinkenlights.ch/ccms/software/bittorrent.html + * Ref: https://github.com/adrian-bl/bitflu/blob/3cb7fe887dbdea8132e4fa36fbbf5f26cf992db3/plugins/Bitflu/20_DownloadBitTorrent.pm#L3403 + * @since 0.9.31 + */ + private static void handleComment(Peer peer, PeerListener listener, byte[] bs, Log log) { + if (log.shouldLog(Log.DEBUG)) + log.debug("Got comment msg from " + peer); + try { + InputStream is = new ByteArrayInputStream(bs); + BDecoder dec = new BDecoder(is); + BEValue bev = dec.bdecodeMap(); + Map map = bev.getMap(); + int type = map.get("msg_type").getInt(); + if (type == 0) { + // request + int num = 20; + BEValue b = map.get("num"); + if (b != null) + num = b.getInt(); + listener.gotCommentReq(peer, num); + } else if (type == 1) { + // response + List list = map.get("comments").getList(); + if (list.isEmpty()) + return; + List comments = new ArrayList(list.size()); + long now = I2PAppContext.getGlobalContext().clock().now(); + for (BEValue li : list) { + Map m = li.getMap(); + String owner = m.get("owner").getString(); + String text = m.get("text").getString(); + int rating = m.get("like").getInt(); + long time = now - (Math.max(0, m.get("timestamp").getInt()) * 1000L); + Comment c = new Comment(text, owner, rating, time, false); + comments.add(c); + } + listener.gotComments(peer, comments); + } else { + if (log.shouldLog(Log.INFO)) + log.info("Unknown comment msg type " + type + " from " + peer); + } + } catch (Exception e) { + if (log.shouldLog(Log.INFO)) + log.info("Comment msg exception from " + peer, e); + //peer.disconnect(false); + } + } + + private static final byte[] COMMENTS_FILTER = new byte[64]; + + /** + * Send comment request + * @since 0.9.31 + */ + public static void sendCommentReq(Peer peer, int num) { + Map map = new HashMap(); + map.put("msg_type", Integer.valueOf(0)); + map.put("num", Integer.valueOf(num)); + map.put("filter", COMMENTS_FILTER); + byte[] payload = BEncoder.bencode(map); + try { + int hisMsgCode = peer.getHandshakeMap().get("m").getMap().get(TYPE_COMMENT).getInt(); + peer.sendExtension(hisMsgCode, payload); + } catch (Exception e) { + // NPE, no caps + } + } + + /** + * Send comments + * Caller must sync on comments + * @param num max to send + * @param comments non-null + * @since 0.9.31 + */ + public static void locked_sendComments(Peer peer, int num, CommentSet comments) { + int toSend = Math.min(num, comments.size()); + if (toSend <= 0) + return; + Map map = new HashMap(); + map.put("msg_type", Integer.valueOf(1)); + List lc = new ArrayList(toSend); + long now = I2PAppContext.getGlobalContext().clock().now(); + int i = 0; + for (Comment c : comments) { + if (i++ >= toSend) + break; + Map mc = new HashMap(); + String s = c.getName(); + mc.put("owner", s != null ? s : ""); + s = c.getText(); + mc.put("text", s != null ? s : ""); + mc.put("like", Integer.valueOf(c.getRating())); + mc.put("timestamp", Long.valueOf((now - c.getTime()) / 1000L)); + lc.add(mc); + } + map.put("comments", lc); + byte[] payload = BEncoder.bencode(map); + try { + int hisMsgCode = peer.getHandshakeMap().get("m").getMap().get(TYPE_COMMENT).getInt(); + peer.sendExtension(hisMsgCode, payload); + } catch (Exception e) { + // NPE, no caps + } + } } diff --git a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java index 9d0b3de32..8e64e6ca6 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java +++ b/apps/i2psnark/java/src/org/klomp/snark/I2PSnarkUtil.java @@ -67,6 +67,8 @@ public class I2PSnarkUtil { private int _startupDelay; private boolean _shouldUseOT; private boolean _shouldUseDHT; + private boolean _enableRatings, _enableComments; + private String _commentsName; private boolean _areFilesPublic; private List _openTrackers; private DHT _dht; @@ -104,6 +106,8 @@ public class I2PSnarkUtil { _shouldUseOT = DEFAULT_USE_OPENTRACKERS; _openTrackers = Collections.emptyList(); _shouldUseDHT = DEFAULT_USE_DHT; + _enableRatings = _enableComments = true; + _commentsName = ""; // This is used for both announce replies and .torrent file downloads, // so it must be available even if not connected to I2CP. // so much for multiple instances @@ -645,6 +649,44 @@ public class I2PSnarkUtil { return _shouldUseDHT; } + /** @since 0.9.31 */ + public void setRatingsEnabled(boolean yes) { + _enableRatings = yes; + } + + /** @since 0.9.31 */ + public boolean ratingsEnabled() { + return _enableRatings; + } + + /** @since 0.9.31 */ + public void setCommentsEnabled(boolean yes) { + _enableComments = yes; + } + + /** @since 0.9.31 */ + public boolean commentsEnabled() { + return _enableComments; + } + + /** @since 0.9.31 */ + public void setCommentsName(String name) { + _commentsName = name; + } + + /** + * @return non-null, "" if none + * @since 0.9.31 + */ + public String getCommentsName() { + return _commentsName; + } + + /** @since 0.9.31 */ + public boolean utCommentsEnabled() { + return _enableRatings || _enableComments; + } + /** * Like DataHelper.toHexString but ensures no loss of leading zero bytes * @since 0.8.4 diff --git a/apps/i2psnark/java/src/org/klomp/snark/Peer.java b/apps/i2psnark/java/src/org/klomp/snark/Peer.java index d89aa6612..1cf1d9aaa 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Peer.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Peer.java @@ -90,6 +90,7 @@ public class Peer implements Comparable //private static final long OPTION_AZMP = 0x1000000000000000l; private long options; private final boolean _isIncoming; + private int _totalCommentsSent; /** * Outgoing connection. @@ -290,7 +291,8 @@ public class Peer implements Comparable int metasize = metainfo != null ? metainfo.getInfoBytes().length : -1; boolean pexAndMetadata = metainfo == null || !metainfo.isPrivate(); boolean dht = util.getDHT() != null; - out.sendExtension(0, ExtensionHandler.getHandshake(metasize, pexAndMetadata, dht, uploadOnly)); + boolean comment = util.utCommentsEnabled(); + out.sendExtension(0, ExtensionHandler.getHandshake(metasize, pexAndMetadata, dht, uploadOnly, comment)); } // Send our bitmap @@ -746,4 +748,14 @@ public class Peer implements Comparable { return PeerCoordinator.getRate(downloaded_old); } + + /** @since 0.9.31 */ + int getTotalCommentsSent() { + return _totalCommentsSent; + } + + /** @since 0.9.31 */ + void setTotalCommentsSent(int count) { + _totalCommentsSent = count; + } } diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java index e0ac005f5..9ca75a73b 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCheckerTask.java @@ -81,7 +81,9 @@ class PeerCheckerTask implements Runnable " interested: " + coordinator.getInterestedUploaders() + " limit: " + uploadLimit + " overBW? " + overBWLimit); DHT dht = _util.getDHT(); + int i = 0; for (Peer peer : peerList) { + i++; // Remove dying peers if (!peer.isConnected()) @@ -226,9 +228,12 @@ class PeerCheckerTask implements Runnable } } peer.retransmitRequests(); - // send PEX - if ((_runCount % 17) == 0 && !peer.isCompleted()) + // send PEX, about every 12 minutes + if (((_runCount + i) % 17) == 0 && !peer.isCompleted()) coordinator.sendPeers(peer); + // send Comment Request, about every 30 minutes + if ( /* comments enabled && */ ((_runCount + i) % 47) == 0) + coordinator.sendCommentReq(peer); // cheap failsafe for seeds connected to seeds, stop pinging and hopefully // the inactive checker (above) will eventually disconnect it if (coordinator.getNeededLength() > 0 || !peer.isCompleted()) @@ -238,7 +243,7 @@ class PeerCheckerTask implements Runnable dht.announce(coordinator.getInfoHash(), peer.getPeerID().getDestHash(), peer.isCompleted()); } - } + } // for peer // Resync actual uploaders value // (can shift a bit by disconnecting peers) diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java index 415ee0d1a..9e914c419 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java @@ -47,6 +47,8 @@ import net.i2p.util.SimpleTimer2; import org.klomp.snark.bencode.BEValue; import org.klomp.snark.bencode.InvalidBEncodingException; +import org.klomp.snark.comments.Comment; +import org.klomp.snark.comments.CommentSet; import org.klomp.snark.dht.DHT; /** @@ -1386,6 +1388,8 @@ class PeerCoordinator implements PeerListener } else if (id == ExtensionHandler.ID_HANDSHAKE) { sendPeers(peer); sendDHT(peer); + if (_util.utCommentsEnabled()) + sendCommentReq(peer); } } @@ -1434,6 +1438,35 @@ class PeerCoordinator implements PeerListener } catch (InvalidBEncodingException ibee) {} } + /** + * Send a commment request message to the peer, if he supports it. + * @since 0.9.31 + */ + void sendCommentReq(Peer peer) { + Map handshake = peer.getHandshakeMap(); + if (handshake == null) + return; + BEValue bev = handshake.get("m"); + if (bev == null) + return; + // TODO if peer hasn't been connected very long, don't bother + // unless forced at handshake time (see above) + try { + if (bev.getMap().get(ExtensionHandler.TYPE_COMMENT) != null) { + int sz = 0; + CommentSet comments = snark.getComments(); + if (comments != null) { + synchronized(comments) { + sz = comments.size(); + } + } + if (sz >= CommentSet.MAX_SIZE) + return; + ExtensionHandler.sendCommentReq(peer, CommentSet.MAX_SIZE - sz); + } + } catch (InvalidBEncodingException ibee) {} + } + /** * Sets the storage after transition out of magnet mode * Snark calls this after we call gotMetaInfo() @@ -1485,6 +1518,40 @@ class PeerCoordinator implements PeerListener // rather than running another thread here. } + /** + * Called when comments are requested via ut_comment + * + * @since 0.9.31 + */ + public void gotCommentReq(Peer peer, int num) { + /* if disabled, return */ + CommentSet comments = snark.getComments(); + if (comments != null) { + int lastSent = peer.getTotalCommentsSent(); + int sz; + synchronized(comments) { + sz = comments.size(); + // only send if we have more than last time + if (sz <= lastSent) + return; + ExtensionHandler.locked_sendComments(peer, num, comments); + } + peer.setTotalCommentsSent(sz); + } + } + + /** + * Called when comments are received via ut_comment + * + * @param comments non-null + * @since 0.9.31 + */ + public void gotComments(Peer peer, List comments) { + /* if disabled, return */ + if (!comments.isEmpty()) + snark.addComments(comments); + } + /** * Called by TrackerClient * @return the Set itself, modifiable, not a copy, caller should clear() diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerListener.java b/apps/i2psnark/java/src/org/klomp/snark/PeerListener.java index ee6801167..07656f6dc 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerListener.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerListener.java @@ -24,6 +24,8 @@ import java.util.List; import net.i2p.data.ByteArray; +import org.klomp.snark.comments.Comment; + /** * Listener for Peer events. */ @@ -215,4 +217,18 @@ interface PeerListener * @since 0.9.2 */ public I2PSnarkUtil getUtil(); + + /** + * Called when comments are requested via ut_comment + * + * @since 0.9.31 + */ + public void gotCommentReq(Peer peer, int num); + + /** + * Called when comments are received via ut_comment + * + * @since 0.9.31 + */ + public void gotComments(Peer peer, List comments); } diff --git a/apps/i2psnark/java/src/org/klomp/snark/Snark.java b/apps/i2psnark/java/src/org/klomp/snark/Snark.java index 350bab829..203f60195 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/Snark.java +++ b/apps/i2psnark/java/src/org/klomp/snark/Snark.java @@ -36,6 +36,10 @@ import net.i2p.data.Destination; import net.i2p.util.Log; import net.i2p.util.SecureFile; +import org.klomp.snark.comments.Comment; +import org.klomp.snark.comments.CommentSet; + + /** * Main Snark program startup class. * @@ -240,6 +244,8 @@ public class Snark private volatile String activity = "Not started"; private long savedUploaded; private long _startedTime; + private CommentSet _comments; + private final Object _commentLock = new Object(); private static final AtomicInteger __RPCID = new AtomicInteger(); private final int _rpcID = __RPCID.incrementAndGet(); @@ -474,6 +480,9 @@ public class Snark */ savedUploaded = (completeListener != null) ? completeListener.getSavedUploaded(this) : 0; + if (completeListener != null) + _comments = completeListener.getSavedComments(this); + if (start) startTorrent(); } @@ -648,6 +657,17 @@ public class Snark savedUploaded = nowUploaded; if (changed && completeListener != null) completeListener.updateStatus(this); + // TODO should save comments at shutdown even if never started... + if (completeListener != null) { + synchronized(_commentLock) { + if (_comments != null) { + synchronized(_comments) { + if (_comments.isModified()) + completeListener.locked_saveComments(this, _comments); + } + } + } + } } if (fast) // HACK: See above if(!fast) @@ -1396,4 +1416,38 @@ public class Snark public long getStartedTime() { return _startedTime; } + + /** + * The current comment set for this torrent. + * Not a copy. + * Caller MUST synch on the returned object for all operations. + * + * @return may be null if none + * @since 0.9.31 + */ + public CommentSet getComments() { + synchronized(_commentLock) { + return _comments; + } + } + + /** + * Add to the current comment set for this torrent, + * creating it if it didn't previously exist. + * + * @return true if the set changed + * @since 0.9.31 + */ + public boolean addComments(List comments) { + synchronized(_commentLock) { + if (_comments == null) { + _comments = new CommentSet(comments); + return true; + } else { + synchronized(_comments) { + return _comments.addAll(comments); + } + } + } + } } diff --git a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java index 039db6af7..cc379357e 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java +++ b/apps/i2psnark/java/src/org/klomp/snark/SnarkManager.java @@ -47,6 +47,8 @@ import net.i2p.util.SimpleTimer2; import net.i2p.util.SystemVersion; import net.i2p.util.Translate; +import org.klomp.snark.comments.Comment; +import org.klomp.snark.comments.CommentSet; import org.klomp.snark.dht.DHT; import org.klomp.snark.dht.KRPC; @@ -113,6 +115,7 @@ public class SnarkManager implements CompleteListener, ClientApp { private static final String CONFIG_FILE_SUFFIX = ".config"; private static final String CONFIG_FILE = "i2psnark" + CONFIG_FILE_SUFFIX; + private static final String COMMENT_FILE_SUFFIX = ".comments.txt.gz"; public static final String PROP_FILES_PUBLIC = "i2psnark.filesPublic"; public static final String PROP_OLD_AUTO_START = "i2snark.autoStart"; // oops public static final String PROP_AUTO_START = "i2psnark.autoStart"; // convert in migration to new config file @@ -133,6 +136,12 @@ public class SnarkManager implements CompleteListener, ClientApp { private static final String PROP_SMART_SORT = "i2psnark.smartSort"; private static final String PROP_LANG = "i2psnark.lang"; private static final String PROP_COUNTRY = "i2psnark.country"; + /** @since 0.9.31 */ + private static final String PROP_RATINGS = "i2psnark.ratings"; + /** @since 0.9.31 */ + private static final String PROP_COMMENTS = "i2psnark.comments"; + /** @since 0.9.31 */ + private static final String PROP_COMMENTS_NAME = "i2psnark.commentsName"; public static final int MIN_UP_BW = 10; public static final int DEFAULT_MAX_UP_BW = 25; @@ -387,7 +396,7 @@ public class SnarkManager implements CompleteListener, ClientApp { * Escapes '<' and '>' before queueing */ public void addMessage(String message) { - addMessageNoEscape(message.replace("<", "<").replace(">", ">")); + addMessageNoEscape(message.replace("&", "&").replace("<", "<").replace(">", ">")); } /** @@ -654,6 +663,53 @@ public class SnarkManager implements CompleteListener, ClientApp { return new File(subdir, hex + CONFIG_FILE_SUFFIX); } + /** + * The conmment file for a torrent + * @param confDir the config directory + * @param ih 20-byte infohash + * @since 0.9.31 + */ + private static File commentFile(File confDir, byte[] ih) { + String hex = I2PSnarkUtil.toHex(ih); + File subdir = new SecureDirectory(confDir, SUBDIR_PREFIX + B64.charAt((ih[0] >> 2) & 0x3f)); + return new File(subdir, hex + COMMENT_FILE_SUFFIX); + } + + /** + * The conmments for a torrent + * @return null if none + * @since 0.9.31 + */ + public CommentSet getSavedComments(Snark snark) { + File com = commentFile(_configDir, snark.getInfoHash()); + if (com.exists()) { + try { + return new CommentSet(com); + } catch (IOException ioe) { + if (_log.shouldLog(Log.WARN)) + _log.warn("Comment load error", ioe); + } + } + return null; + } + + /** + * Save the conmments for a torrent + * Caller must synchronize on comments. + * + * @param comments non-null + * @since 0.9.31 + */ + public void locked_saveComments(Snark snark, CommentSet comments) { + File com = commentFile(_configDir, snark.getInfoHash()); + try { + comments.save(com); + } catch (IOException ioe) { + if (_log.shouldLog(Log.WARN)) + _log.warn("Comment save error", ioe); + } + } + /** * Extract the info hash from a config file name * @return null for invalid name @@ -730,6 +786,12 @@ public class SnarkManager implements CompleteListener, ClientApp { // no, so we can switch default to true later //if (!_config.containsKey(PROP_USE_DHT)) // _config.setProperty(PROP_USE_DHT, Boolean.toString(I2PSnarkUtil.DEFAULT_USE_DHT)); + if (!_config.containsKey(PROP_RATINGS)) + _config.setProperty(PROP_RATINGS, "true"); + if (!_config.containsKey(PROP_COMMENTS)) + _config.setProperty(PROP_COMMENTS, "true"); + if (!_config.containsKey(PROP_COMMENTS_NAME)) + _config.setProperty(PROP_COMMENTS_NAME, ""); updateConfig(); } @@ -831,6 +893,9 @@ public class SnarkManager implements CompleteListener, ClientApp { // careful, so we can switch default to true later _util.setUseDHT(Boolean.parseBoolean(_config.getProperty(PROP_USE_DHT, Boolean.toString(I2PSnarkUtil.DEFAULT_USE_DHT)))); + _util.setRatingsEnabled(Boolean.parseBoolean(_config.getProperty(PROP_RATINGS, "true"))); + _util.setCommentsEnabled(Boolean.parseBoolean(_config.getProperty(PROP_COMMENTS, "true"))); + _util.setCommentsName(_config.getProperty(PROP_COMMENTS_NAME, "")); getDataDir().mkdirs(); initTrackerMap(); } @@ -853,13 +918,13 @@ public class SnarkManager implements CompleteListener, ClientApp { String startDelay, String pageSize, String seedPct, String eepHost, String eepPort, String i2cpHost, String i2cpPort, String i2cpOpts, String upLimit, String upBW, boolean useOpenTrackers, boolean useDHT, String theme, - String lang) { + String lang, boolean enableRatings, boolean enableComments, String commentName) { synchronized(_configLock) { locked_updateConfig(dataDir, filesPublic, autoStart, smartSort,refreshDelay, startDelay, pageSize, seedPct, eepHost, eepPort, i2cpHost, i2cpPort, i2cpOpts, upLimit, upBW, useOpenTrackers, useDHT, theme, - lang); + lang, enableRatings, enableComments, commentName); } } @@ -867,7 +932,7 @@ public class SnarkManager implements CompleteListener, ClientApp { String startDelay, String pageSize, String seedPct, String eepHost, String eepPort, String i2cpHost, String i2cpPort, String i2cpOpts, String upLimit, String upBW, boolean useOpenTrackers, boolean useDHT, String theme, - String lang) { + String lang, boolean enableRatings, boolean enableComments, String commentName) { boolean changed = false; boolean interruptMonitor = false; //if (eepHost != null) { @@ -1138,6 +1203,37 @@ public class SnarkManager implements CompleteListener, ClientApp { _util.setUseDHT(useDHT); changed = true; } + if (_util.ratingsEnabled() != enableRatings) { + _config.setProperty(PROP_RATINGS, Boolean.toString(enableRatings)); + if (enableRatings) + addMessage(_t("Enabled Ratings.")); + else + addMessage(_t("Disabled Ratings.")); + _util.setRatingsEnabled(enableRatings); + changed = true; + } + if (_util.commentsEnabled() != enableComments) { + _config.setProperty(PROP_COMMENTS, Boolean.toString(enableComments)); + if (enableComments) + addMessage(_t("Enabled Comments.")); + else + addMessage(_t("Disabled Comments.")); + _util.setCommentsEnabled(enableComments); + changed = true; + } + if (commentName == null) { + commentName = ""; + } else { + commentName = commentName.replaceAll("[\n\r<>#;]", ""); + if (commentName.length() > Comment.MAX_NAME_LEN) + commentName = commentName.substring(0, Comment.MAX_NAME_LEN); + } + if (!_util.getCommentsName().equals(commentName)) { + _config.setProperty(PROP_COMMENTS_NAME, commentName); + addMessage(_t("Comments name set to {0}.", commentName)); + _util.setCommentsName(commentName); + changed = true; + } if (theme != null) { if(!theme.equals(_config.getProperty(PROP_THEME))) { _config.setProperty(PROP_THEME, theme); @@ -1936,7 +2032,9 @@ public class SnarkManager implements CompleteListener, ClientApp { private void removeTorrentStatus(Snark snark) { byte[] ih = snark.getInfoHash(); File conf = configFile(_configDir, ih); + File comm = commentFile(_configDir, ih); synchronized (_configLock) { + comm.delete(); boolean ok = conf.delete(); if (ok) { if (_log.shouldInfo()) @@ -2659,6 +2757,15 @@ public class SnarkManager implements CompleteListener, ClientApp { if (count % 8 == 0) { try { Thread.sleep(20); } catch (InterruptedException ie) {} } + } else { + CommentSet cs = snark.getComments(); + if (cs != null) { + synchronized(cs) { + if (cs.isModified()) { + locked_saveComments(snark, cs); + } + } + } } } if (_util.connected()) { diff --git a/apps/i2psnark/java/src/org/klomp/snark/UpdateRunner.java b/apps/i2psnark/java/src/org/klomp/snark/UpdateRunner.java index 4f9dff55a..fca254317 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/UpdateRunner.java +++ b/apps/i2psnark/java/src/org/klomp/snark/UpdateRunner.java @@ -11,6 +11,9 @@ import net.i2p.update.*; import net.i2p.util.Log; import net.i2p.util.SimpleTimer2; +import org.klomp.snark.comments.CommentSet; + + /** * The downloader for router signed updates. * @@ -299,6 +302,16 @@ class UpdateRunner implements UpdateTask, CompleteListener { return _smgr.getSavedUploaded(snark); } + /** @since 0.9.31 */ + public CommentSet getSavedComments(Snark snark) { + return _smgr.getSavedComments(snark); + } + + /** @since 0.9.31 */ + public void locked_saveComments(Snark snark, CommentSet comments) { + _smgr.locked_saveComments(snark, comments); + } + //////// end CompleteListener methods private static String linkify(String url) { diff --git a/apps/i2psnark/java/src/org/klomp/snark/comments/Comment.java b/apps/i2psnark/java/src/org/klomp/snark/comments/Comment.java new file mode 100644 index 000000000..8ac662efb --- /dev/null +++ b/apps/i2psnark/java/src/org/klomp/snark/comments/Comment.java @@ -0,0 +1,219 @@ +/* + * Released into the public domain + * with no warranty of any kind, either expressed or implied. + */ +package org.klomp.snark.comments; + +import java.util.concurrent.atomic.AtomicInteger; + +import net.i2p.I2PAppContext; +import net.i2p.data.DataHelper; + +/** + * Store comments + * + * @since 0.9.31 + */ +public class Comment implements Comparable { + + private final String text, name; + // seconds since 1/1/2005 + private final int time; + private final byte rating; + private final boolean byMe; + private boolean hidden; + private static final AtomicInteger _id = new AtomicInteger(); + private final int id = _id.incrementAndGet(); + + public static final int MAX_NAME_LEN = 32; + // same as IRC, more or less + private static final int MAX_TEXT_LEN = 512; + private static final int BUCKET_SIZE = 10*60*1000; + private static final long TIME_SHRINK = 1000L; + // 1/1/2005 + private static final long TIME_OFFSET = 1104537600000L; + + /** + * My comment, now + * + * @param text may be null, will be truncated to max length, newlines replaced with spaces + * @param name may be null, will be truncated to max length, newlines and commas removed + * @param rating 0-5 + */ + public Comment(String text, String name, int rating) { + this(text, name, rating, I2PAppContext.getGlobalContext().clock().now(), true); + } + + /** + * @param text may be null, will be truncated to max length, newlines replaced with spaces + * @param name may be null, will be truncated to max length, newlines and commas removed + * @param time java time (ms) + * @param rating 0-5 + */ + public Comment(String text, String name, int rating, long time, boolean isMine) { + if (text != null) { + text = text.trim(); + text = text.replaceAll("[\r\n]", " "); + if (text.length() == 0) + text = null; + else if (text.length() > MAX_TEXT_LEN) + text = text.substring(0, MAX_TEXT_LEN); + } + this.text = text; + if (name != null) { + name = name.trim(); + // comma because it's not last in the persistent string + name = name.replaceAll("[,\r\n]", ""); + if (name.length() == 0) + name = null; + else if (name.length() > MAX_NAME_LEN) + name = name.substring(0, MAX_NAME_LEN); + } + this.name = name; + if (rating < 0 || rating > 5) + rating = 0; + else if (rating > 5) + rating = 5; + this.rating = (byte) rating; + if (time < TIME_OFFSET) { + time = TIME_OFFSET; + } else { + long now = I2PAppContext.getGlobalContext().clock().now(); + if (time > now) + time = now; + } + this.time = (int) ((time - TIME_OFFSET) / TIME_SHRINK); + this.byMe = isMine; + } + + public String getText() { return text; } + + public String getName() { return name; } + + public int getRating() { return rating; } + + /** java time (ms) */ + public long getTime() { return (time * TIME_SHRINK) + TIME_OFFSET; } + + public boolean isMine() { return byMe; } + + public boolean isHidden() { return hidden; } + + void setHidden() { hidden = true; } + + /** + * A unique ID that may be used to delete this comment from + * the CommentSet via remove(int). NOT persisted across restarts. + */ + public int getID() { return id; } + + /** + * reverse + */ + public int compareTo(Comment c) { + if (time > c.time) + return -1; + if (time < c.time) + return 1; + // arbitrary sort below here + if (rating != c.rating) + return c.rating - rating; + if (name != null || c.name != null) { + if (name == null) + return 1; + if (c.name == null) + return -1; + int rv = name.compareTo(c.name); + if (rv != 0) + return rv; + } + if (text != null || c.text != null) { + if (text == null) + return 1; + if (c.text == null) + return -1; + int rv = text.compareTo(c.text); + if (rv != 0) + return rv; + } + return 0; + } + + /** + * @return time,rating,mine,hidden,name,text + */ + public String toPersistentString() { + StringBuilder buf = new StringBuilder(); + buf.append(getTime()); + buf.append(','); + buf.append(Byte.toString(rating)); + buf.append(','); + buf.append(byMe ? "1" : "0"); + buf.append(','); + buf.append(hidden ? "1" : "0"); + buf.append(','); + if (name != null) + buf.append(name); + buf.append(','); + if (text != null) + buf.append(text); + return buf.toString(); + } + + /** + * @return null if can't be parsed + */ + public static Comment fromPersistentString(String s) { + String[] ss = DataHelper.split(s, ",", 6); + if (ss.length != 6) + return null; + try { + long t = Long.parseLong(ss[0]); + int r = Integer.parseInt(ss[1]); + boolean m = !ss[2].equals("0"); + boolean h = !ss[3].equals("0"); + Comment rv = new Comment(ss[5], ss[4], r, t, m); + if (h) + rv.setHidden(); + return rv; + } catch (NumberFormatException nfe) { + return null; + } + } + + + @Override + public int hashCode() { + return time / (BUCKET_SIZE / (int) TIME_SHRINK); + } + + /** + * Comments in the same 10-minute bucket and otherwise equal + * are considered equal. This will result in duplicates + * near the border. + */ + @Override + public boolean equals(Object o) { + if (o == null) return false; + if (!(o instanceof Comment)) return false; + Comment c = (Comment) o; + return rating == c.rating && + eq(text, c.text) && + eq(name, c.name) && + hashCode() == c.hashCode(); + } + + /** + * Ignores timestamp + * @param c non-null + */ + public boolean equalsIgnoreTimestamp(Comment c) { + return rating == c.rating && + eq(text, c.text) && + eq(name, c.name); + } + + private static boolean eq(String lhs, String rhs) { + return (lhs == null && rhs == null) || (lhs != null && lhs.equals(rhs)); + } +} diff --git a/apps/i2psnark/java/src/org/klomp/snark/comments/CommentSet.java b/apps/i2psnark/java/src/org/klomp/snark/comments/CommentSet.java new file mode 100644 index 000000000..eed4eff31 --- /dev/null +++ b/apps/i2psnark/java/src/org/klomp/snark/comments/CommentSet.java @@ -0,0 +1,352 @@ +/* + * Released into the public domain + * with no warranty of any kind, either expressed or implied. + */ +package org.klomp.snark.comments; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStreamReader; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.util.AbstractSet; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +import net.i2p.util.SecureFileOutputStream; + +/** + * Store comments. + * + * Optimized for fast checking of duplicates, and retrieval of ratings. + * Removes are not really removed, only marked as hidden, so + * they don't reappear. + * Duplicates are detected based on an approximate time range. + * Max size of both elements and total text length is enforced. + * + * Supports persistence via save() and File constructor. + * + * NOT THREAD SAFE except for iterating AFTER the iterator() call. + * + * @since 0.9.31 + */ +public class CommentSet extends AbstractSet { + + private final HashMap> map; + private int size, realSize; + private int myRating; + private int totalRating; + private int ratingSize; + private int totalTextSize; + private long latestCommentTime; + private boolean modified; + + public static final int MAX_SIZE = 256; + + // Comment.java enforces max text length of 512, but + // we don't want 256*512 in memory per-torrent, so + // track and enforce separately. + // Assume most comments are short or null. + private static final int MAX_TOTAL_TEXT_LEN = MAX_SIZE * 16; + + public CommentSet() { + super(); + map = new HashMap>(4); + } + + public CommentSet(Collection coll) { + super(); + map = new HashMap>(coll.size()); + addAll(coll); + } + + /** + * File must be gzipped. + * Need not be sorted. + * See Comment.toPersistentString() for format. + */ + public CommentSet(File file) throws IOException { + this(); + BufferedReader br = null; + try { + br = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(file)), "UTF-8")); + String line = null; + while ( (line = br.readLine()) != null) { + Comment c = Comment.fromPersistentString(line); + if (c != null) + add(c); + } + } finally { + if (br != null) try { br.close(); } catch (IOException ioe) {} + } + modified = false; + } + + /** + * File will be gzipped. + * Not sorted, includes hidden. + * See Comment.toPersistentString() for format. + * Sets isModified() to false. + */ + public void save(File file) throws IOException { + PrintWriter out = null; + try { + out = new PrintWriter(new OutputStreamWriter(new GZIPOutputStream(new SecureFileOutputStream(file)), "UTF-8")); + for (List l : map.values()) { + for (Comment c : l) { + out.println(c.toPersistentString()); + } + } + if (out.checkError()) + throw new IOException("Failed write to " + file); + modified = false; + } finally { + if (out != null) out.close(); + } + } + + /** + * Max length for strings enforced in Comment.java. + * Max total length for strings enforced here. + * Enforces max size for set + */ + @Override + public boolean add(Comment c) { + if (realSize >= MAX_SIZE && !c.isMine()) + return false; + String s = c.getText(); + if (s != null && totalTextSize + s.length() > MAX_TOTAL_TEXT_LEN) + return false; + // If isMine and no text and rating changed, don't bother + if (c.isMine() && c.getText() == null && c.getRating() == myRating) + return false; + Integer hc = Integer.valueOf(c.hashCode()); + List list = map.get(hc); + if (list == null) { + list = Collections.singletonList(c); + map.put(hc, list); + addStats(c); + return true; + } + if (list.contains(c)) + return false; + if (list.size() == 1) { + // presume unmodifiable singletonList + List nlist = new ArrayList(2); + nlist.add(list.get(0)); + map.put(hc, nlist); + list = nlist; + } + list.add(c); + // If isMine and no text and comment changed, remove old ones + if (c.isMine() && c.getText() == null) + removeMyOldRatings(c.getID()); + addStats(c); + return true; + } + + /** + * Only hides the comment, doesn't really remove it. + * @return true if present and not previously hidden + */ + @Override + public boolean remove(Object o) { + if (o == null || !(o instanceof Comment)) + return false; + Comment c = (Comment) o; + Integer hc = Integer.valueOf(c.hashCode()); + List list = map.get(hc); + if (list == null) + return false; + int i = list.indexOf(c); + if (i >= 0) { + Comment cc = list.get(i); + if (!cc.isHidden()) { + removeStats(cc); + cc.setHidden(); + return true; + } + } + return false; + } + + /** + * Remove the id as retrieved from Comment.getID(). + * Only hides the comment, doesn't really remove it. + * This is for the UI. + * + * @return true if present and not previously hidden + */ + public boolean remove(int id) { + // not the most efficient but should be rare. + for (List l : map.values()) { + for (Comment c : l) { + if (c.getID() == id) { + return remove(c); + } + } + } + return false; + } + + /** + * Remove all ratings of mine with empty comments, + * except the ID specified. + */ + private void removeMyOldRatings(int exceptID) { + for (List l : map.values()) { + for (Comment c : l) { + if (c.isMine() && c.getText() == null && c.getID() != exceptID && !c.isHidden()) { + removeStats(c); + c.setHidden(); + } + } + } + } + + /** may be hidden */ + private void addStats(Comment c) { + realSize++; + if (!c.isHidden()) { + size++; + int r = c.getRating(); + if (r > 0) { + if (c.isMine()) { + myRating = r; + } else { + totalRating += r; + ratingSize++; + } + } + long time = c.getTime(); + if (time > latestCommentTime) + latestCommentTime = time; + } + String t = c.getText(); + if (t != null) + totalTextSize += t.length(); + modified = true; + } + + /** call before setting hidden */ + private void removeStats(Comment c) { + if (!c.isHidden()) { + size--; + int r = c.getRating(); + if (r > 0) { + if (c.isMine()) { + if (myRating == r) + myRating = 0; + } else { + totalRating -= r; + ratingSize--; + } + } + modified = true; + } + } + + /** + * Is not adjusted if the latest comment wasn't hidden but is then hidden. + * @return the timestamp of the most recent non-hidden comment + */ + public long getLatestCommentTime() { return latestCommentTime; } + + /** + * @return true if modified since instantiation + */ + public boolean isModified() { return modified; } + + /** + * @return 0 if none, or 1-5 + */ + public int getMyRating() { return myRating; } + + /** + * @return Number of ratings making up the average rating + */ + public int getRatingCount() { return ratingSize; } + + /** + * @return 0 if none, or 1-5 + */ + public double getAverageRating() { + if (ratingSize <= 0) + return 0.0d; + return totalRating / (double) ratingSize; + } + + /** + * Actually clears everything, including hidden. + * Resets ratings to zero. + */ + @Override + public void clear() { + if (realSize > 0) { + modified = true; + realSize = 0; + map.clear(); + size = 0; + myRating = 0; + totalRating = 0; + ratingSize = 0; + totalTextSize = 0; + } + } + + /** + * May be more than what the iterator returns, + * we do additional deduping in the iterator. + * + * @return the non-hidden size + */ + public int size() { + return size; + } + + /** + * Will be in reverse-sort order, i.e. newest-first. + * The returned iterator is thread-safe after this call. + * Changes after this call will not be reflected in the iterator. + * iter.remove() has no effect on the underlying set. + * Hidden comments not included. + * + * Returned values may be less than indicated in size() + * due to additional deduping in the iterator. + */ + public Iterator iterator() { + List list = new ArrayList(size); + for (List l : map.values()) { + int hc = l.get(0).hashCode(); + List prevList = map.get(Integer.valueOf(hc - 1)); + for (Comment c : l) { + if (!c.isHidden()) { + // additional deduping at boundary + if (prevList != null) { + boolean dup = false; + for (Comment pc : prevList) { + if (c.equalsIgnoreTimestamp(pc)) { + dup = true; + break; + } + } + if (dup) + continue; + } + list.add(c); + } + } + } + Collections.sort(list); + return list.iterator(); + } +} diff --git a/apps/i2psnark/java/src/org/klomp/snark/comments/package.html b/apps/i2psnark/java/src/org/klomp/snark/comments/package.html new file mode 100644 index 000000000..c4df9251b --- /dev/null +++ b/apps/i2psnark/java/src/org/klomp/snark/comments/package.html @@ -0,0 +1,7 @@ + + +

+Data structures to support ut_comment, since 0.9.31. +

+ + diff --git a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java index e5300e824..4a73cb27b 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java +++ b/apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java @@ -1150,9 +1150,13 @@ public class I2PSnarkServlet extends BasicServlet { //String openTrackers = req.getParameter("openTrackers"); String theme = req.getParameter("theme"); String lang = req.getParameter("lang"); + boolean ratings = req.getParameter("ratings") != null; + boolean comments = req.getParameter("comments") != null; + String commentsName = req.getParameter("nofilter_commentsName"); _manager.updateConfig(dataDir, filesPublic, autoStart, smartSort, refreshDel, startupDel, pageSize, seedPct, eepHost, eepPort, i2cpHost, i2cpPort, i2cpOpts, - upLimit, upBW, useOpenTrackers, useDHT, theme, lang); + upLimit, upBW, useOpenTrackers, useDHT, theme, + lang, ratings, comments, commentsName); // update servlet try { setResourceBase(_manager.getDataDir()); diff --git a/history.txt b/history.txt index 244954d76..70f984a1b 100644 --- a/history.txt +++ b/history.txt @@ -1,3 +1,7 @@ +2017-05-05 zzz + * Blockfile: Move from i2p.jar to addressbook.jar + * i2psnark: Initial support for ut_comment, no UI yet + * 2017-05-03 0.9.30 released 2017-04-30 zzz diff --git a/router/java/src/net/i2p/router/RouterVersion.java b/router/java/src/net/i2p/router/RouterVersion.java index 87e5bffca..282c18b42 100644 --- a/router/java/src/net/i2p/router/RouterVersion.java +++ b/router/java/src/net/i2p/router/RouterVersion.java @@ -18,7 +18,7 @@ public class RouterVersion { /** deprecated */ public final static String ID = "Monotone"; public final static String VERSION = CoreVersion.VERSION; - public final static long BUILD = 1; + public final static long BUILD = 2; /** for example "-test" */ public final static String EXTRA = "";