From d5c80505722a06caf9281006b33ef9385f7e6a05 Mon Sep 17 00:00:00 2001 From: Zlatin Balevsky Date: Fri, 14 Feb 2020 00:37:07 +0000 Subject: [PATCH] wip on separate hashlist storage #35 --- .../muwire/core/download/Downloader.groovy | 5 +- .../core/files/BasePersisterService.groovy | 57 ++++++++++++++++--- .../core/files/FileDownloadedEvent.groovy | 2 + .../muwire/core/files/FileHashedEvent.groovy | 2 + .../muwire/core/files/FileLoadedEvent.groovy | 2 + .../com/muwire/core/files/FileManager.groovy | 2 +- .../muwire/core/files/HasherService.groovy | 3 +- .../core/files/PersisterFolderService.groovy | 35 +++++++++--- .../muwire/core/update/UpdateClient.groovy | 2 +- .../java/com/muwire/core/DownloadedFile.java | 4 +- .../main/java/com/muwire/core/SharedFile.java | 36 +++--------- .../views/com/muwire/gui/MainFrameView.groovy | 3 +- .../java/com/muwire/webui/FileManager.java | 2 - 13 files changed, 103 insertions(+), 52 deletions(-) diff --git a/core/src/main/groovy/com/muwire/core/download/Downloader.groovy b/core/src/main/groovy/com/muwire/core/download/Downloader.groovy index f4df59c6..b8aeb186 100644 --- a/core/src/main/groovy/com/muwire/core/download/Downloader.groovy +++ b/core/src/main/groovy/com/muwire/core/download/Downloader.groovy @@ -405,8 +405,9 @@ public class Downloader { } eventBus.publish( new FileDownloadedEvent( - downloadedFile : new DownloadedFile(file.getCanonicalFile(), getInfoHash(), pieceSizePow2, successfulDestinations), - downloader : Downloader.this)) + downloadedFile : new DownloadedFile(file.getCanonicalFile(), getInfoHash().getRoot(), pieceSizePow2, successfulDestinations), + downloader : Downloader.this, + infoHash: getInfoHash())) } endpoint?.close() diff --git a/core/src/main/groovy/com/muwire/core/files/BasePersisterService.groovy b/core/src/main/groovy/com/muwire/core/files/BasePersisterService.groovy index fcb8313a..356bb419 100644 --- a/core/src/main/groovy/com/muwire/core/files/BasePersisterService.groovy +++ b/core/src/main/groovy/com/muwire/core/files/BasePersisterService.groovy @@ -51,13 +51,13 @@ abstract class BasePersisterService extends Service{ if (json.sources != null) { List sources = (List)json.sources Set sourceSet = sources.stream().map({ d -> new Destination(d.toString())}).collect Collectors.toSet() - DownloadedFile df = new DownloadedFile(file, ih, pieceSize, sourceSet) + DownloadedFile df = new DownloadedFile(file, ih.getRoot(), pieceSize, sourceSet) df.setComment(json.comment) - return new FileLoadedEvent(loadedFile : df) + return new FileLoadedEvent(loadedFile : df, infoHash: ih) } - SharedFile sf = new SharedFile(file, ih, pieceSize) + SharedFile sf = new SharedFile(file, ih.getRoot(), pieceSize) sf.setComment(json.comment) if (json.downloaders != null) sf.getDownloaders().addAll(json.downloaders) @@ -71,18 +71,61 @@ abstract class BasePersisterService extends Service{ sf.hit(searcher, timestamp, query) } } - return new FileLoadedEvent(loadedFile: sf) + return new FileLoadedEvent(loadedFile: sf, infoHash: ih) } + + protected static FileLoadedEvent fromJsonLite(json) { + if (json.file == null || json.length == null || json.root == null) + throw new IllegalArgumentException() + + def file = new File(DataUtil.readi18nString(Base64.decode(json.file))) + file = file.getCanonicalFile() + if (!file.exists() || file.isDirectory()) + return null + long length = Long.valueOf(json.length) + if (length != file.length()) + return null + + byte[] root = Base64.decode(json.root) + InfoHash ih = new InfoHash(root) + + int pieceSize = 0 + if (json.pieceSize != null) + pieceSize = json.pieceSize + + if (json.sources != null) { + List sources = (List)json.sources + Set sourceSet = sources.stream().map({ d -> new Destination(d.toString())}).collect Collectors.toSet() + DownloadedFile df = new DownloadedFile(file, ih.getRoot(), pieceSize, sourceSet) + df.setComment(json.comment) + return new FileLoadedEvent(loadedFile : df, infoHash: ih) + } + + + SharedFile sf = new SharedFile(file, ih.getRoot(), pieceSize) + sf.setComment(json.comment) + if (json.downloaders != null) + sf.getDownloaders().addAll(json.downloaders) + if (json.searchers != null) { + json.searchers.each { + Persona searcher = null + if (it.searcher != null) + searcher = new Persona(new ByteArrayInputStream(Base64.decode(it.searcher))) + long timestamp = it.timestamp + String query = it.query + sf.hit(searcher, timestamp, query) + } + } + return new FileLoadedEvent(loadedFile: sf, infoHash: ih) + } protected static toJson(SharedFile sf) { def json = [:] json.file = sf.getB64EncodedFileName() json.length = sf.getCachedLength() - InfoHash ih = sf.getInfoHash() - json.infoHash = sf.getB64EncodedHashRoot() + json.root = Base64.encode(sf.getRoot()) json.pieceSize = sf.getPieceSize() - json.hashList = sf.getB64EncodedHashList() json.comment = sf.getComment() json.hits = sf.getHits() json.downloaders = sf.getDownloaders() diff --git a/core/src/main/groovy/com/muwire/core/files/FileDownloadedEvent.groovy b/core/src/main/groovy/com/muwire/core/files/FileDownloadedEvent.groovy index aa9de96c..5c5d8281 100644 --- a/core/src/main/groovy/com/muwire/core/files/FileDownloadedEvent.groovy +++ b/core/src/main/groovy/com/muwire/core/files/FileDownloadedEvent.groovy @@ -2,6 +2,7 @@ package com.muwire.core.files import com.muwire.core.DownloadedFile import com.muwire.core.Event +import com.muwire.core.InfoHash import com.muwire.core.download.Downloader import net.i2p.data.Destination @@ -9,4 +10,5 @@ import net.i2p.data.Destination class FileDownloadedEvent extends Event { Downloader downloader DownloadedFile downloadedFile + InfoHash infoHash } diff --git a/core/src/main/groovy/com/muwire/core/files/FileHashedEvent.groovy b/core/src/main/groovy/com/muwire/core/files/FileHashedEvent.groovy index 07948faf..cadaaa99 100644 --- a/core/src/main/groovy/com/muwire/core/files/FileHashedEvent.groovy +++ b/core/src/main/groovy/com/muwire/core/files/FileHashedEvent.groovy @@ -1,11 +1,13 @@ package com.muwire.core.files import com.muwire.core.Event +import com.muwire.core.InfoHash import com.muwire.core.SharedFile class FileHashedEvent extends Event { SharedFile sharedFile + InfoHash infoHash String error @Override diff --git a/core/src/main/groovy/com/muwire/core/files/FileLoadedEvent.groovy b/core/src/main/groovy/com/muwire/core/files/FileLoadedEvent.groovy index 0dc31f0c..537bbb4b 100644 --- a/core/src/main/groovy/com/muwire/core/files/FileLoadedEvent.groovy +++ b/core/src/main/groovy/com/muwire/core/files/FileLoadedEvent.groovy @@ -1,10 +1,12 @@ package com.muwire.core.files import com.muwire.core.Event +import com.muwire.core.InfoHash import com.muwire.core.SharedFile class FileLoadedEvent extends Event { SharedFile loadedFile + InfoHash infoHash String source } diff --git a/core/src/main/groovy/com/muwire/core/files/FileManager.groovy b/core/src/main/groovy/com/muwire/core/files/FileManager.groovy index 3b86392b..b206b89a 100644 --- a/core/src/main/groovy/com/muwire/core/files/FileManager.groovy +++ b/core/src/main/groovy/com/muwire/core/files/FileManager.groovy @@ -75,7 +75,7 @@ class FileManager { private void addToIndex(SharedFile sf) { log.info("Adding shared file " + sf.getFile()) - InfoHash infoHash = sf.getInfoHash() + InfoHash infoHash = new InfoHash(sf.getRoot()) Set existing = rootToFiles.get(infoHash) if (existing == null) { log.info("adding new root") diff --git a/core/src/main/groovy/com/muwire/core/files/HasherService.groovy b/core/src/main/groovy/com/muwire/core/files/HasherService.groovy index f9aff41e..79856547 100644 --- a/core/src/main/groovy/com/muwire/core/files/HasherService.groovy +++ b/core/src/main/groovy/com/muwire/core/files/HasherService.groovy @@ -65,7 +65,8 @@ class HasherService { } else { eventBus.publish new FileHashingEvent(hashingFile: f) def hash = hasher.hashFile f - eventBus.publish new FileHashedEvent(sharedFile: new SharedFile(f, hash, FileHasher.getPieceSize(f.length()))) + eventBus.publish new FileHashedEvent(sharedFile: new SharedFile(f, hash.getRoot(), FileHasher.getPieceSize(f.length())), + infoHash : hash) } } } diff --git a/core/src/main/groovy/com/muwire/core/files/PersisterFolderService.groovy b/core/src/main/groovy/com/muwire/core/files/PersisterFolderService.groovy index f58b5db0..b9977651 100644 --- a/core/src/main/groovy/com/muwire/core/files/PersisterFolderService.groovy +++ b/core/src/main/groovy/com/muwire/core/files/PersisterFolderService.groovy @@ -56,17 +56,17 @@ class PersisterFolderService extends BasePersisterService { } void onFileHashedEvent(FileHashedEvent hashedEvent) { - persistFile(hashedEvent.sharedFile) + persistFile(hashedEvent.sharedFile, hashedEvent.infoHash) } void onFileDownloadedEvent(FileDownloadedEvent downloadedEvent) { if (core.getMuOptions().getShareDownloadedFiles()) { - persistFile(downloadedEvent.downloadedFile) + persistFile(downloadedEvent.downloadedFile, downloadedEvent.infoHash) } } /** - * Get rid of the json of unshared files + * Get rid of the json and hashlists of unshared files * @param unsharedEvent */ void onFileUnsharedEvent(FileUnsharedEvent unsharedEvent) { @@ -75,12 +75,17 @@ class PersisterFolderService extends BasePersisterService { if(jsonFile.isFile()){ jsonFile.delete() } + def hashListPath = getHashListPath(unsharedEvent.unsharedFile) + def hashListFile = hashListPath.toFile() + if (hashListFile.isFile()) + hashListFile.delete() } + void onFileLoadedEvent(FileLoadedEvent loadedEvent) { if(loadedEvent.source == "PersisterService"){ log.info("Migrating persisted file from PersisterService: " + loadedEvent.loadedFile.file.absolutePath.toString()) - persistFile(loadedEvent.loadedFile) + persistFile(loadedEvent.loadedFile, loadedEvent.infoHash) } } @@ -109,10 +114,12 @@ class PersisterFolderService extends BasePersisterService { int loaded = 0 def slurper = new JsonSlurper() Files.walk(location.toPath()) - .filter({ it.fileName.endsWith(".json") }) + .filter({ + it.getFileName().toString().endsWith(".json") + }) .forEach({ def parsed = slurper.parse it.toFile() - def event = fromJson parsed + def event = fromJsonLite parsed if (event == null) return log.fine("loaded file $event.loadedFile.file") @@ -125,7 +132,7 @@ class PersisterFolderService extends BasePersisterService { listener.publish(new AllFilesLoadedEvent()) } - private void persistFile(SharedFile sf) { + private void persistFile(SharedFile sf, InfoHash ih) { persisterExecutor.submit({ def jsonPath = getJsonPath(sf) @@ -136,7 +143,10 @@ class PersisterFolderService extends BasePersisterService { json = JsonOutput.toJson(json) writer.println json } - log.fine("Time(ms) to write json: " + (System.currentTimeMillis() - startTime)) + + def hashListPath = getHashListPath(sf) + hashListPath.toFile().bytes = ih.hashList + log.fine("Time(ms) to write json+hashList: " + (System.currentTimeMillis() - startTime)) } as Runnable) } private Path getJsonPath(SharedFile sf){ @@ -147,4 +157,13 @@ class PersisterFolderService extends BasePersisterService { pathHash.substring(CUT_LENGTH) + ".json" ) } + + private Path getHashListPath(SharedFile sf) { + def pathHash = sf.getB64PathHash() + return Paths.get( + location.getAbsolutePath(), + pathHash.substring(0, CUT_LENGTH), + pathHash.substring(CUT_LENGTH) + ".hashlist" + ) + } } diff --git a/core/src/main/groovy/com/muwire/core/update/UpdateClient.groovy b/core/src/main/groovy/com/muwire/core/update/UpdateClient.groovy index 00248e04..9175db04 100644 --- a/core/src/main/groovy/com/muwire/core/update/UpdateClient.groovy +++ b/core/src/main/groovy/com/muwire/core/update/UpdateClient.groovy @@ -83,7 +83,7 @@ class UpdateClient { } void onFileDownloadedEvent(FileDownloadedEvent e) { - if (e.downloadedFile.infoHash != updateInfoHash) + if (e.infoHash != updateInfoHash) return updateDownloading = false eventBus.publish(new UpdateDownloadedEvent(version : version, signer : signer, text : text)) diff --git a/core/src/main/java/com/muwire/core/DownloadedFile.java b/core/src/main/java/com/muwire/core/DownloadedFile.java index 54feadf2..7a7faaa2 100644 --- a/core/src/main/java/com/muwire/core/DownloadedFile.java +++ b/core/src/main/java/com/muwire/core/DownloadedFile.java @@ -10,9 +10,9 @@ public class DownloadedFile extends SharedFile { private final Set sources; - public DownloadedFile(File file, InfoHash infoHash, int pieceSize, Set sources) + public DownloadedFile(File file, byte[] root, int pieceSize, Set sources) throws IOException { - super(file, infoHash, pieceSize); + super(file, root, pieceSize); this.sources = sources; } diff --git a/core/src/main/java/com/muwire/core/SharedFile.java b/core/src/main/java/com/muwire/core/SharedFile.java index 5f238960..8021a494 100644 --- a/core/src/main/java/com/muwire/core/SharedFile.java +++ b/core/src/main/java/com/muwire/core/SharedFile.java @@ -5,6 +5,7 @@ import java.io.IOException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -18,7 +19,7 @@ import net.i2p.data.Base64; public class SharedFile { private final File file; - private final InfoHash infoHash; + private final byte[] root; private final int pieceSize; private final String cachedPath; @@ -26,29 +27,18 @@ public class SharedFile { private String b64PathHash; private final String b64EncodedFileName; - private final String b64EncodedHashRoot; - private final List b64EncodedHashList; private volatile String comment; private final Set downloaders = Collections.synchronizedSet(new HashSet<>()); private final Set searches = Collections.synchronizedSet(new HashSet<>()); - public SharedFile(File file, InfoHash infoHash, int pieceSize) throws IOException { + public SharedFile(File file, byte[] root, int pieceSize) throws IOException { this.file = file; - this.infoHash = infoHash; + this.root = root; this.pieceSize = pieceSize; this.cachedPath = file.getAbsolutePath(); this.cachedLength = file.length(); this.b64EncodedFileName = Base64.encode(DataUtil.encodei18nString(file.toString())); - this.b64EncodedHashRoot = Base64.encode(infoHash.getRoot()); - - List b64List = new ArrayList(); - byte[] tmp = new byte[32]; - for (int i = 0; i < infoHash.getHashList().length / 32; i++) { - System.arraycopy(infoHash.getHashList(), i * 32, tmp, 0, 32); - b64List.add(Base64.encode(tmp)); - } - this.b64EncodedHashList = b64List; } public File getFile() { @@ -56,7 +46,7 @@ public class SharedFile { } public byte[] getPathHash() throws NoSuchAlgorithmException { - var digester = MessageDigest.getInstance("SHA-256"); + MessageDigest digester = MessageDigest.getInstance("SHA-256"); digester.update(file.getAbsolutePath().getBytes()); return digester.digest(); } @@ -68,8 +58,8 @@ public class SharedFile { return b64PathHash; } - public InfoHash getInfoHash() { - return infoHash; + public byte[] getRoot() { + return root; } public int getPieceSize() { @@ -89,14 +79,6 @@ public class SharedFile { return b64EncodedFileName; } - public String getB64EncodedHashRoot() { - return b64EncodedHashRoot; - } - - public List getB64EncodedHashList() { - return b64EncodedHashList; - } - public String getCachedPath() { return cachedPath; } @@ -135,7 +117,7 @@ public class SharedFile { @Override public int hashCode() { - return file.hashCode() ^ infoHash.hashCode(); + return file.hashCode() ^ Arrays.hashCode(root); } @Override @@ -143,7 +125,7 @@ public class SharedFile { if (!(o instanceof SharedFile)) return false; SharedFile other = (SharedFile)o; - return file.equals(other.file) && infoHash.equals(other.infoHash); + return file.equals(other.file) && Arrays.equals(root, other.root); } public static class SearchEntry { diff --git a/gui/griffon-app/views/com/muwire/gui/MainFrameView.groovy b/gui/griffon-app/views/com/muwire/gui/MainFrameView.groovy index 9783fb8a..1f5f1b64 100644 --- a/gui/griffon-app/views/com/muwire/gui/MainFrameView.groovy +++ b/gui/griffon-app/views/com/muwire/gui/MainFrameView.groovy @@ -35,6 +35,7 @@ import javax.swing.tree.TreePath import com.muwire.core.Constants import com.muwire.core.Core +import com.muwire.core.InfoHash import com.muwire.core.MuWireSettings import com.muwire.core.SharedFile import com.muwire.core.download.Downloader @@ -289,7 +290,7 @@ class MainFrameView { closureColumn(header : "Comments", preferredWidth : 50, type : Boolean, read : {it.getComment() != null}) closureColumn(header : "Certified", preferredWidth : 50, type : Boolean, read : { Core core = application.context.get("core") - core.certificateManager.hasLocalCertificate(it.getInfoHash()) + core.certificateManager.hasLocalCertificate(new InfoHash(it.getRoot())) }) closureColumn(header : "Search Hits", preferredWidth: 50, type : Integer, read : {it.getHits()}) closureColumn(header : "Downloaders", preferredWidth: 50, type : Integer, read : {it.getDownloaders().size()}) diff --git a/webui/src/main/java/com/muwire/webui/FileManager.java b/webui/src/main/java/com/muwire/webui/FileManager.java index b894dc34..3e11b8ac 100644 --- a/webui/src/main/java/com/muwire/webui/FileManager.java +++ b/webui/src/main/java/com/muwire/webui/FileManager.java @@ -21,7 +21,6 @@ import com.muwire.core.files.FileTree; import com.muwire.core.files.FileTreeCallback; import com.muwire.core.files.FileUnsharedEvent; import com.muwire.core.files.UICommentEvent; -import com.muwire.core.files.UIPersistFilesEvent; import com.muwire.core.util.DataUtil; import net.i2p.data.Base64; @@ -132,7 +131,6 @@ public class FileManager { core.getEventBus().publish(event); } } - core.getEventBus().publish(new UIPersistFilesEvent()); Util.pause(); }