AGentooCat/i2p.i2p
1
0
Fork 0
forked from I2P_Developers/i2p.i2p
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Profiles: Limit storage and memory usage

Browse Source 
Write profiles to disk more often
Delete old profiles on disk more often
Reduce max age of profiles
Limit age of profiles read in at startup based on downtime
Limit total profiles read in at startup
Change loaded profiles from a Set to a List for efficiency
Log tweaks
This commit is contained in:
zzz
2023-02-12 10:11:46 -05:00
parent 7a75ea4bef
commit afe236c62c
3 changed files with 72 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
router/java/src/net/i2p/router/peermanager/PeerManager.java
View File

@@ -57,8 +57,9 @@ class PeerManager {
static final long REORGANIZE_TIME_LONG = 351*1000;
/** After first two hours of uptime ~= 246 */
static final int REORGANIZES_PER_DAY = (int) (24*60*60*1000L / REORGANIZE_TIME_LONG);
private static final long STORE_TIME = 19*60*60*1000;
private static final long EXPIRE_AGE = 3*24*60*60*1000;
private static final long STORE_TIME = 2*60*60*1000;
// for profiles stored to disk
private static final long EXPIRE_AGE = 3*60*60*1000;
public static final String TRACKED_CAPS = "" +
FloodfillNetworkDatabaseFacade.CAPABILITY_FLOODFILL +
@@ -136,8 +137,11 @@ class PeerManager {
try {
_log.debug("Periodic profile store start");
storeProfiles();
if (shouldDecay)
_persistenceHelper.deleteOldProfiles(EXPIRE_AGE);
if (shouldDecay) {
int count = _persistenceHelper.deleteOldProfiles(EXPIRE_AGE);
if (count > 0 && _log.shouldInfo())
_log.info("Deleted " + count + " old profiles");
}
_log.debug("Periodic profile store end");
} catch (Throwable t) {
_log.log(Log.CRIT, "Error storing profiles", t);
@@ -158,17 +162,25 @@ class PeerManager {
// Don't overwrite disk profiles when testing
if (_context.commSystem().isDummy())
return;
long now = _context.clock().now();
long cutoff = now - EXPIRE_AGE;
// lock in case shutdown bumps into periodic store
if (!_storeLock.compareAndSet(false, true))
return;
int i = 0;
int total;
try {
Set<Hash> peers = selectPeers();
total = peers.size();
for (Hash peer : peers) {
storeProfile(peer);
if (storeProfile(peer, cutoff))
i++;
}
} finally {
_storeLock.set(false);
}
if (_log.shouldInfo())
_log.info("Stored " + i + " out of " + total + " profiles");
}
/** @since 0.8.8 */
@@ -183,12 +195,18 @@ class PeerManager {
return _organizer.selectAllPeers();
}
void storeProfile(Hash peer) {
if (peer == null) return;
/**
* @param cutoff only store if last successful send newer than this (absolute time)
* @return success
*/
private boolean storeProfile(Hash peer, long cutoff) {
PeerProfile prof = _organizer.getProfile(peer);
if (prof == null) return;
if (true)
_persistenceHelper.writeProfile(prof);
if (prof == null) return false;
if (prof.getLastSendSuccessful() > cutoff) {
if (_persistenceHelper.writeProfile(prof))
return true;
}
return false;
}
/**
@@ -219,12 +237,12 @@ class PeerManager {
* This may take a long time - 30 seconds or more
*/
void loadProfiles() {
Set<PeerProfile> profiles = _persistenceHelper.readProfiles();
List<PeerProfile> profiles = _persistenceHelper.readProfiles();
for (PeerProfile prof : profiles) {
_organizer.addProfile(prof);
if (_log.shouldLog(Log.DEBUG))
_log.debug("Profile for " + prof.getPeer().toBase64() + " loaded");
}
if (_log.shouldInfo())
_log.info("Loaded " + profiles.size() + " profiles");
}
/**

4
router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java
View File

@@ -385,6 +385,8 @@ public class ProfileOrganizer {
}
/**
* Only for console
*
* @return true if successful, false if not found
*/
public boolean exportProfile(Hash profile, OutputStream out) throws IOException {
@@ -1550,7 +1552,7 @@ public class ProfileOrganizer {
organizer.setUs(Hash.FAKE_HASH);
ProfilePersistenceHelper helper = new ProfilePersistenceHelper(ctx);
for (int i = 0; i < args.length; i++) {
PeerProfile profile = helper.readProfile(new java.io.File(args[i]));
PeerProfile profile = helper.readProfile(new java.io.File(args[i]), 0);
if (profile == null) {
System.err.println("Could not load profile " + args[i]);
continue;

71
router/java/src/net/i2p/router/peermanager/ProfilePersistenceHelper.java
View File

@@ -9,11 +9,10 @@ import java.io.InputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
@@ -25,6 +24,7 @@ import net.i2p.util.FileUtil;
import net.i2p.util.Log;
import net.i2p.util.SecureDirectory;
import net.i2p.util.SecureFileOutputStream;
import net.i2p.util.SystemVersion;
/**
* Write profiles to disk at shutdown,
@@ -47,14 +47,8 @@ class ProfilePersistenceHelper {
private static final int MIN_NAME_LENGTH = PREFIX.length() + 44 + OLD_SUFFIX.length();
private static final String DIR_PREFIX = "p";
private static final String B64 = Base64.ALPHABET_I2P;
/**
* If we haven't been able to get a message through to the peer in this much time,
* drop the profile. They may reappear, but if they do, their config may
* have changed (etc).
*
*/
private static final long EXPIRE_AGE = 15*24*60*60*1000;
// Max to read in at startup
private static final int LIMIT_PROFILES = SystemVersion.isSlow() ? 1000 : 4000;
private final File _profileDir;
private Hash _us;
@@ -77,25 +71,21 @@ class ProfilePersistenceHelper {
/**
* write out the data from the profile to the file
* @return success
*/
public void writeProfile(PeerProfile profile) {
if (isExpired(profile.getLastSendSuccessful()))
return;
public boolean writeProfile(PeerProfile profile) {
File f = pickFile(profile);
long before = _context.clock().now();
OutputStream fos = null;
try {
fos = new BufferedOutputStream(new GZIPOutputStream(new SecureFileOutputStream(f)));
writeProfile(profile, fos, false);
} catch (IOException ioe) {
_log.error("Error writing profile to " + f);
return false;
} finally {
if (fos != null) try { fos.close(); } catch (IOException ioe) {}
}
long delay = _context.clock().now() - before;
if (_log.shouldLog(Log.DEBUG))
_log.debug("Writing the profile to " + f.getName() + " took " + delay + "ms");
return true;
}
/**
@@ -206,18 +196,29 @@ class ProfilePersistenceHelper {
buf.append(NL);
}
public Set<PeerProfile> readProfiles() {
long start = _context.clock().now();
public List<PeerProfile> readProfiles() {
long start = System.currentTimeMillis();
long down = _context.router().getEstimatedDowntime();
long cutoff = down < 15*24*60*60*1000L ? start - down - 24*60*60*1000 : start;
List<File> files = selectFiles();
Set<PeerProfile> profiles = new HashSet<PeerProfile>(files.size());
if (files.size() > LIMIT_PROFILES)
Collections.shuffle(files, _context.random());
List<PeerProfile> profiles = new ArrayList<PeerProfile>(Math.min(LIMIT_PROFILES, files.size()));
int count = 0;
for (File f : files) {
PeerProfile profile = readProfile(f);
if (profile != null)
if (count >= LIMIT_PROFILES) {
f.delete();
continue;
}
PeerProfile profile = readProfile(f, cutoff);
if (profile != null) {
profiles.add(profile);
count++;
}
}
long duration = _context.clock().now() - start;
long duration = System.currentTimeMillis() - start;
if (_log.shouldLog(Log.DEBUG))
_log.debug("Loading " + profiles.size() + " took " + duration + "ms");
_log.debug("Loading " + count + " profiles took " + duration + "ms");
return profiles;
}
@@ -263,9 +264,10 @@ class ProfilePersistenceHelper {
/**
* Delete profile files with timestamps older than 'age' ago
* @return number deleted
* @since 0.9.28
*/
public void deleteOldProfiles(long age) {
public int deleteOldProfiles(long age) {
long cutoff = System.currentTimeMillis() - age;
List<File> files = selectFiles();
int i = 0;
@@ -279,15 +281,18 @@ class ProfilePersistenceHelper {
}
if (_log.shouldWarn())
_log.warn("Deleted " + i + " old profiles");
return i;
}
private boolean isExpired(long lastSentToSuccessfully) {
long timeSince = _context.clock().now() - lastSentToSuccessfully;
return (timeSince > EXPIRE_AGE);
}
/**
* @param cutoff delete and return null if older than this (absolute time)
*/
@SuppressWarnings("deprecation")
public PeerProfile readProfile(File file) {
public PeerProfile readProfile(File file, long cutoff) {
if (file.lastModified() < cutoff) {
file.delete();
return null;
}
Hash peer = getHash(file.getName());
try {
if (peer == null) {
@@ -300,7 +305,7 @@ class ProfilePersistenceHelper {
loadProps(props, file);
long lastSentToSuccessfully = getLong(props, "lastSentToSuccessfully");
if (isExpired(lastSentToSuccessfully)) {
if (lastSentToSuccessfully <= cutoff) {
if (_log.shouldLog(Log.INFO))
_log.info("Dropping old profile " + file.getName() +
", since we haven't heard from them in a long time");