idk/muwire
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

wip on separate hashlist storage #35

Browse Source
This commit is contained in:
Zlatin Balevsky
2020-02-14 00:37:07 +00:00
parent 83546d68d2
commit d5c8050572
13 changed files with 103 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
core/src/main/groovy/com/muwire/core/download/Downloader.groovy
View File

@@ -405,8 +405,9 @@ public class Downloader {
}
eventBus.publish(
new FileDownloadedEvent(
downloadedFile : new DownloadedFile(file.getCanonicalFile(), getInfoHash(), pieceSizePow2, successfulDestinations),
downloader : Downloader.this))
downloadedFile : new DownloadedFile(file.getCanonicalFile(), getInfoHash().getRoot(), pieceSizePow2, successfulDestinations),
downloader : Downloader.this,
infoHash: getInfoHash()))
}
endpoint?.close()

57
core/src/main/groovy/com/muwire/core/files/BasePersisterService.groovy
View File

@@ -51,13 +51,13 @@ abstract class BasePersisterService extends Service{
if (json.sources != null) {
List sources = (List)json.sources
Set<Destination> sourceSet = sources.stream().map({ d -> new Destination(d.toString())}).collect Collectors.toSet()
DownloadedFile df = new DownloadedFile(file, ih, pieceSize, sourceSet)
DownloadedFile df = new DownloadedFile(file, ih.getRoot(), pieceSize, sourceSet)
df.setComment(json.comment)
return new FileLoadedEvent(loadedFile : df)
return new FileLoadedEvent(loadedFile : df, infoHash: ih)
}
SharedFile sf = new SharedFile(file, ih, pieceSize)
SharedFile sf = new SharedFile(file, ih.getRoot(), pieceSize)
sf.setComment(json.comment)
if (json.downloaders != null)
sf.getDownloaders().addAll(json.downloaders)
@@ -71,18 +71,61 @@ abstract class BasePersisterService extends Service{
sf.hit(searcher, timestamp, query)
}
}
return new FileLoadedEvent(loadedFile: sf)
return new FileLoadedEvent(loadedFile: sf, infoHash: ih)
}
protected static FileLoadedEvent fromJsonLite(json) {
if (json.file == null || json.length == null || json.root == null)
throw new IllegalArgumentException()
def file = new File(DataUtil.readi18nString(Base64.decode(json.file)))
file = file.getCanonicalFile()
if (!file.exists() || file.isDirectory())
return null
long length = Long.valueOf(json.length)
if (length != file.length())
return null
byte[] root = Base64.decode(json.root)
InfoHash ih = new InfoHash(root)
int pieceSize = 0
if (json.pieceSize != null)
pieceSize = json.pieceSize
if (json.sources != null) {
List sources = (List)json.sources
Set<Destination> sourceSet = sources.stream().map({ d -> new Destination(d.toString())}).collect Collectors.toSet()
DownloadedFile df = new DownloadedFile(file, ih.getRoot(), pieceSize, sourceSet)
df.setComment(json.comment)
return new FileLoadedEvent(loadedFile : df, infoHash: ih)
}
SharedFile sf = new SharedFile(file, ih.getRoot(), pieceSize)
sf.setComment(json.comment)
if (json.downloaders != null)
sf.getDownloaders().addAll(json.downloaders)
if (json.searchers != null) {
json.searchers.each {
Persona searcher = null
if (it.searcher != null)
searcher = new Persona(new ByteArrayInputStream(Base64.decode(it.searcher)))
long timestamp = it.timestamp
String query = it.query
sf.hit(searcher, timestamp, query)
}
}
return new FileLoadedEvent(loadedFile: sf, infoHash: ih)
}
protected static toJson(SharedFile sf) {
def json = [:]
json.file = sf.getB64EncodedFileName()
json.length = sf.getCachedLength()
InfoHash ih = sf.getInfoHash()
json.infoHash = sf.getB64EncodedHashRoot()
json.root = Base64.encode(sf.getRoot())
json.pieceSize = sf.getPieceSize()
json.hashList = sf.getB64EncodedHashList()
json.comment = sf.getComment()
json.hits = sf.getHits()
json.downloaders = sf.getDownloaders()

2
core/src/main/groovy/com/muwire/core/files/FileDownloadedEvent.groovy
View File

@@ -2,6 +2,7 @@ package com.muwire.core.files
import com.muwire.core.DownloadedFile
import com.muwire.core.Event
import com.muwire.core.InfoHash
import com.muwire.core.download.Downloader
import net.i2p.data.Destination
@@ -9,4 +10,5 @@ import net.i2p.data.Destination
class FileDownloadedEvent extends Event {
Downloader downloader
DownloadedFile downloadedFile
InfoHash infoHash
}

2
core/src/main/groovy/com/muwire/core/files/FileHashedEvent.groovy
View File

@@ -1,11 +1,13 @@
package com.muwire.core.files
import com.muwire.core.Event
import com.muwire.core.InfoHash
import com.muwire.core.SharedFile
class FileHashedEvent extends Event {
SharedFile sharedFile
InfoHash infoHash
String error
@Override

2
core/src/main/groovy/com/muwire/core/files/FileLoadedEvent.groovy
View File

@@ -1,10 +1,12 @@
package com.muwire.core.files
import com.muwire.core.Event
import com.muwire.core.InfoHash
import com.muwire.core.SharedFile
class FileLoadedEvent extends Event {
SharedFile loadedFile
InfoHash infoHash
String source
}

2
core/src/main/groovy/com/muwire/core/files/FileManager.groovy
View File

@@ -75,7 +75,7 @@ class FileManager {
private void addToIndex(SharedFile sf) {
log.info("Adding shared file " + sf.getFile())
InfoHash infoHash = sf.getInfoHash()
InfoHash infoHash = new InfoHash(sf.getRoot())
Set<SharedFile> existing = rootToFiles.get(infoHash)
if (existing == null) {
log.info("adding new root")

3
core/src/main/groovy/com/muwire/core/files/HasherService.groovy
View File

@@ -65,7 +65,8 @@ class HasherService {
} else {
eventBus.publish new FileHashingEvent(hashingFile: f)
def hash = hasher.hashFile f
eventBus.publish new FileHashedEvent(sharedFile: new SharedFile(f, hash, FileHasher.getPieceSize(f.length())))
eventBus.publish new FileHashedEvent(sharedFile: new SharedFile(f, hash.getRoot(), FileHasher.getPieceSize(f.length())),
infoHash : hash)
}
}
}

35
core/src/main/groovy/com/muwire/core/files/PersisterFolderService.groovy
View File

@@ -56,17 +56,17 @@ class PersisterFolderService extends BasePersisterService {
}
void onFileHashedEvent(FileHashedEvent hashedEvent) {
persistFile(hashedEvent.sharedFile)
persistFile(hashedEvent.sharedFile, hashedEvent.infoHash)
}
void onFileDownloadedEvent(FileDownloadedEvent downloadedEvent) {
if (core.getMuOptions().getShareDownloadedFiles()) {
persistFile(downloadedEvent.downloadedFile)
persistFile(downloadedEvent.downloadedFile, downloadedEvent.infoHash)
}
}
/**
* Get rid of the json of unshared files
* Get rid of the json and hashlists of unshared files
* @param unsharedEvent
*/
void onFileUnsharedEvent(FileUnsharedEvent unsharedEvent) {
@@ -75,12 +75,17 @@ class PersisterFolderService extends BasePersisterService {
if(jsonFile.isFile()){
jsonFile.delete()
}
def hashListPath = getHashListPath(unsharedEvent.unsharedFile)
def hashListFile = hashListPath.toFile()
if (hashListFile.isFile())
hashListFile.delete()
}
void onFileLoadedEvent(FileLoadedEvent loadedEvent) {
if(loadedEvent.source == "PersisterService"){
log.info("Migrating persisted file from PersisterService: "
+ loadedEvent.loadedFile.file.absolutePath.toString())
persistFile(loadedEvent.loadedFile)
persistFile(loadedEvent.loadedFile, loadedEvent.infoHash)
}
}
@@ -109,10 +114,12 @@ class PersisterFolderService extends BasePersisterService {
int loaded = 0
def slurper = new JsonSlurper()
Files.walk(location.toPath())
.filter({ it.fileName.endsWith(".json") })
.filter({
it.getFileName().toString().endsWith(".json")
})
.forEach({
def parsed = slurper.parse it.toFile()
def event = fromJson parsed
def event = fromJsonLite parsed
if (event == null) return
log.fine("loaded file $event.loadedFile.file")
@@ -125,7 +132,7 @@ class PersisterFolderService extends BasePersisterService {
listener.publish(new AllFilesLoadedEvent())
}
private void persistFile(SharedFile sf) {
private void persistFile(SharedFile sf, InfoHash ih) {
persisterExecutor.submit({
def jsonPath = getJsonPath(sf)
@@ -136,7 +143,10 @@ class PersisterFolderService extends BasePersisterService {
json = JsonOutput.toJson(json)
writer.println json
}
log.fine("Time(ms) to write json: " + (System.currentTimeMillis() - startTime))
def hashListPath = getHashListPath(sf)
hashListPath.toFile().bytes = ih.hashList
log.fine("Time(ms) to write json+hashList: " + (System.currentTimeMillis() - startTime))
} as Runnable)
}
private Path getJsonPath(SharedFile sf){
@@ -147,4 +157,13 @@ class PersisterFolderService extends BasePersisterService {
pathHash.substring(CUT_LENGTH) + ".json"
)
}
private Path getHashListPath(SharedFile sf) {
def pathHash = sf.getB64PathHash()
return Paths.get(
location.getAbsolutePath(),
pathHash.substring(0, CUT_LENGTH),
pathHash.substring(CUT_LENGTH) + ".hashlist"
)
}
}

2
core/src/main/groovy/com/muwire/core/update/UpdateClient.groovy
View File

@@ -83,7 +83,7 @@ class UpdateClient {
}
void onFileDownloadedEvent(FileDownloadedEvent e) {
if (e.downloadedFile.infoHash != updateInfoHash)
if (e.infoHash != updateInfoHash)
return
updateDownloading = false
eventBus.publish(new UpdateDownloadedEvent(version : version, signer : signer, text : text))

4
core/src/main/java/com/muwire/core/DownloadedFile.java
View File

@@ -10,9 +10,9 @@ public class DownloadedFile extends SharedFile {
private final Set<Destination> sources;
public DownloadedFile(File file, InfoHash infoHash, int pieceSize, Set<Destination> sources)
public DownloadedFile(File file, byte[] root, int pieceSize, Set<Destination> sources)
throws IOException {
super(file, infoHash, pieceSize);
super(file, root, pieceSize);
this.sources = sources;
}

36
core/src/main/java/com/muwire/core/SharedFile.java
View File

@@ -5,6 +5,7 @@ import java.io.IOException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
@@ -18,7 +19,7 @@ import net.i2p.data.Base64;
public class SharedFile {
private final File file;
private final InfoHash infoHash;
private final byte[] root;
private final int pieceSize;
private final String cachedPath;
@@ -26,29 +27,18 @@ public class SharedFile {
private String b64PathHash;
private final String b64EncodedFileName;
private final String b64EncodedHashRoot;
private final List<String> b64EncodedHashList;
private volatile String comment;
private final Set<String> downloaders = Collections.synchronizedSet(new HashSet<>());
private final Set<SearchEntry> searches = Collections.synchronizedSet(new HashSet<>());
public SharedFile(File file, InfoHash infoHash, int pieceSize) throws IOException {
public SharedFile(File file, byte[] root, int pieceSize) throws IOException {
this.file = file;
this.infoHash = infoHash;
this.root = root;
this.pieceSize = pieceSize;
this.cachedPath = file.getAbsolutePath();
this.cachedLength = file.length();
this.b64EncodedFileName = Base64.encode(DataUtil.encodei18nString(file.toString()));
this.b64EncodedHashRoot = Base64.encode(infoHash.getRoot());
List<String> b64List = new ArrayList<String>();
byte[] tmp = new byte[32];
for (int i = 0; i < infoHash.getHashList().length / 32; i++) {
System.arraycopy(infoHash.getHashList(), i * 32, tmp, 0, 32);
b64List.add(Base64.encode(tmp));
}
this.b64EncodedHashList = b64List;
}
public File getFile() {
@@ -56,7 +46,7 @@ public class SharedFile {
}
public byte[] getPathHash() throws NoSuchAlgorithmException {
var digester = MessageDigest.getInstance("SHA-256");
MessageDigest digester = MessageDigest.getInstance("SHA-256");
digester.update(file.getAbsolutePath().getBytes());
return digester.digest();
}
@@ -68,8 +58,8 @@ public class SharedFile {
return b64PathHash;
}
public InfoHash getInfoHash() {
return infoHash;
public byte[] getRoot() {
return root;
}
public int getPieceSize() {
@@ -89,14 +79,6 @@ public class SharedFile {
return b64EncodedFileName;
}
public String getB64EncodedHashRoot() {
return b64EncodedHashRoot;
}
public List<String> getB64EncodedHashList() {
return b64EncodedHashList;
}
public String getCachedPath() {
return cachedPath;
}
@@ -135,7 +117,7 @@ public class SharedFile {
@Override
public int hashCode() {
return file.hashCode() ^ infoHash.hashCode();
return file.hashCode() ^ Arrays.hashCode(root);
}
@Override
@@ -143,7 +125,7 @@ public class SharedFile {
if (!(o instanceof SharedFile))
return false;
SharedFile other = (SharedFile)o;
return file.equals(other.file) && infoHash.equals(other.infoHash);
return file.equals(other.file) && Arrays.equals(root, other.root);
}
public static class SearchEntry {

3
gui/griffon-app/views/com/muwire/gui/MainFrameView.groovy
View File

@@ -35,6 +35,7 @@ import javax.swing.tree.TreePath
import com.muwire.core.Constants
import com.muwire.core.Core
import com.muwire.core.InfoHash
import com.muwire.core.MuWireSettings
import com.muwire.core.SharedFile
import com.muwire.core.download.Downloader
@@ -289,7 +290,7 @@ class MainFrameView {
closureColumn(header : "Comments", preferredWidth : 50, type : Boolean, read : {it.getComment() != null})
closureColumn(header : "Certified", preferredWidth : 50, type : Boolean, read : {
Core core = application.context.get("core")
core.certificateManager.hasLocalCertificate(it.getInfoHash())
core.certificateManager.hasLocalCertificate(new InfoHash(it.getRoot()))
})
closureColumn(header : "Search Hits", preferredWidth: 50, type : Integer, read : {it.getHits()})
closureColumn(header : "Downloaders", preferredWidth: 50, type : Integer, read : {it.getDownloaders().size()})

2
webui/src/main/java/com/muwire/webui/FileManager.java
View File

@@ -21,7 +21,6 @@ import com.muwire.core.files.FileTree;
import com.muwire.core.files.FileTreeCallback;
import com.muwire.core.files.FileUnsharedEvent;
import com.muwire.core.files.UICommentEvent;
import com.muwire.core.files.UIPersistFilesEvent;
import com.muwire.core.util.DataUtil;
import net.i2p.data.Base64;
@@ -132,7 +131,6 @@ public class FileManager {
core.getEventBus().publish(event);
}
}
core.getEventBus().publish(new UIPersistFilesEvent());
Util.pause();
}