From 3a4bfc9c077f7a2aa6b108a2386dcb90232aab9d Mon Sep 17 00:00:00 2001 From: zzz Date: Sun, 28 Aug 2022 15:07:30 -0400 Subject: [PATCH] Util: Add option to gzip router logs Primarily for devs. No UI. remove shutdown hook ID --- core/java/src/net/i2p/util/FileLogWriter.java | 78 ++++++++++++++++++- core/java/src/net/i2p/util/LogManager.java | 46 +++++++++-- 2 files changed, 114 insertions(+), 10 deletions(-) diff --git a/core/java/src/net/i2p/util/FileLogWriter.java b/core/java/src/net/i2p/util/FileLogWriter.java index dddce27f7..c9b0d7021 100644 --- a/core/java/src/net/i2p/util/FileLogWriter.java +++ b/core/java/src/net/i2p/util/FileLogWriter.java @@ -9,11 +9,19 @@ package net.i2p.util; * */ +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; import java.io.BufferedWriter; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; +import java.util.zip.GZIPOutputStream; + +import net.i2p.data.DataHelper; /** * File-based log writer thread that pulls log records from the LogManager, @@ -93,12 +101,31 @@ class FileLogWriter extends LogWriter { * @since 0.9.19 renamed from closeFile() */ protected void closeWriter() { + closeWriter(_currentFile, false); + } + + /** + * Gzip the closed file + * + * @param threadGzipper if true, spin off a thread + * @since 0.9.55 + */ + private void closeWriter(File currentFile, boolean threadGzipper) { Writer out = _currentOut; if (out != null) { try { out.close(); } catch (IOException ioe) {} } + if (_manager.shouldGzip() && currentFile != null && currentFile.length() >= _manager.getMinGzipSize()) { + Thread gzipper = new Gzipper(currentFile); + if (threadGzipper) { + gzipper.setPriority(Thread.MIN_PRIORITY); + gzipper.start(); // rotate + } else { + gzipper.run(); // shutdown + } + } } /** @@ -107,6 +134,7 @@ class FileLogWriter extends LogWriter { * Caller must synch */ private void rotateFile() { + File old = _currentFile; File f = getNextFile(); _currentFile = f; _numBytesInCurrentFile = 0; @@ -125,7 +153,9 @@ class FileLogWriter extends LogWriter { //System.exit(0); } } - closeWriter(); + closeWriter(old, true); + if (_manager.shouldGzip()) + (new File(f.getPath() + ".gz")).delete(); try { _currentOut = new BufferedWriter(new OutputStreamWriter(new SecureFileOutputStream(f), "UTF-8")); } catch (IOException ioe) { @@ -180,7 +210,8 @@ class FileLogWriter extends LogWriter { f = new File(base, replace(pattern, i)); else f = new File(replace(pattern, i)); - if (!f.exists()) { + // check for file or file.gz + if (!f.exists() && !(_manager.shouldGzip() && (new File(f.getPath() + ".gz").exists()))) { _rotationNum = i; return f; } @@ -197,7 +228,18 @@ class FileLogWriter extends LogWriter { if (oldest == null) { oldest = f; } else { - if (f.lastModified() < oldest.lastModified()) { + // set file or file.gz for last mod check + File ff, oo; + if (!_manager.shouldGzip() || f.exists()) + ff = f; + else + ff = new File(f.getPath() + ".gz"); + if (!_manager.shouldGzip() || oldest.exists()) + oo = oldest; + else + oo = new File(oldest.getPath() + ".gz"); + + if (ff.lastModified() < oo.lastModified()) { _rotationNum = i; oldest = f; } @@ -218,4 +260,34 @@ class FileLogWriter extends LogWriter { } return buf.toString(); } + + /** + * @since 0.9.55 + */ + private static class Gzipper extends I2PAppThread { + private final File _f; + + public Gzipper(File f) { + super("Log file compressor"); + _f = f; + } + + public void run() { + File to = new File(_f.getPath() + ".gz"); + InputStream in = null; + OutputStream out = null; + try { + in = new BufferedInputStream(new FileInputStream(_f)); + out = new BufferedOutputStream(new GZIPOutputStream(new SecureFileOutputStream(to))); + DataHelper.copy(in, out); + } catch (IOException ioe) { + System.out.println("Error compressing log file " + _f); + } finally { + if (in != null) try { in.close(); } catch (IOException ioe) {} + if (out != null) try { out.close(); } catch (IOException ioe) {} + to.setLastModified(_f.lastModified()); + _f.delete(); + } + } + } } diff --git a/core/java/src/net/i2p/util/LogManager.java b/core/java/src/net/i2p/util/LogManager.java index 7c2df1a31..9c6d511c6 100644 --- a/core/java/src/net/i2p/util/LogManager.java +++ b/core/java/src/net/i2p/util/LogManager.java @@ -38,6 +38,8 @@ import net.i2p.data.DataHelper; * This also fires off a LogWriter thread that pulls pending records off and * writes them where appropriate. * + * As of 0.9.41, this class may be overridden via I2PAppContext.setLogManager() + * */ public class LogManager implements Flushable { public final static String CONFIG_LOCATION_PROP = "loggerConfigLocation"; @@ -65,6 +67,10 @@ public class LogManager implements Flushable { private static final String PROP_DUP = "logger.dropDuplicates"; /** @since 0.9.18 */ private static final String PROP_FLUSH = "logger.flushInterval"; + /** @since 0.9.56 */ + private static final String PROP_GZIP = "logger.gzip"; + /** @since 0.9.56 */ + private static final String PROP_MIN_GZIP_SIZE = "logger.minGzipSize"; public final static String PROP_RECORD_PREFIX = "logger.record."; public final static String DEFAULT_FORMAT = DATE + " " + PRIORITY + " [" + THREAD + "] " + CLASS + ": " + MESSAGE; @@ -79,6 +85,9 @@ public class LogManager implements Flushable { public final static String DEFAULT_DEFAULTLEVEL = Log.STR_ERROR; public final static String DEFAULT_ONSCREENLEVEL = Log.STR_CRIT; private static final int MIN_FILESIZE_LIMIT = 16*1024; + private final static boolean DEFAULT_GZIP = false; + private static final int DEFAULT_MIN_GZIP_SIZE = 64*1024; + private final I2PAppContext _context; private final Log _log; @@ -133,6 +142,8 @@ public class LogManager implements Flushable { private final AtomicLong _droppedRecords = new AtomicLong(); // in seconds private int _flushInterval = (int) (LogWriter.FLUSH_INTERVAL / 1000); + private boolean _gzip; + private long _minGzipSize; private boolean _alreadyNoticedMissingConfig; @@ -452,6 +463,17 @@ public class LogManager implements Flushable { String str = config.getProperty(PROP_DUP); _dropDuplicates = str == null || Boolean.parseBoolean(str); + str = config.getProperty(PROP_GZIP); + _gzip = str != null ? Boolean.parseBoolean(str) : DEFAULT_GZIP; + if (_gzip) { + _minGzipSize = DEFAULT_MIN_GZIP_SIZE; + try { + str = config.getProperty(PROP_MIN_GZIP_SIZE); + if (str != null) + _minGzipSize = Long.parseLong(str); + } catch (NumberFormatException nfe) {} + } + //if (_log.shouldLog(Log.DEBUG)) // _log.debug("Log set to use the base log file as " + _baseLogfilename); @@ -673,6 +695,20 @@ public class LogManager implements Flushable { return _rotationLimit; } + /** + * @since 0.9.56 + */ + boolean shouldGzip() { + return _gzip; + } + + /** + * @since 0.9.56 + */ + long getMinGzipSize() { + return _gzip ? _minGzipSize : Long.MAX_VALUE; + } + /** @return success */ public synchronized boolean saveConfig() { Properties props = createConfig(); @@ -712,6 +748,8 @@ public class LogManager implements Flushable { rv.setProperty(PROP_DISPLAYONSCREENLEVEL, Log.toLevelString(_onScreenLimit)); rv.setProperty(PROP_CONSOLEBUFFERSIZE, Integer.toString(_consoleBufferSize)); rv.setProperty(PROP_FLUSH, Integer.toString(_flushInterval)); + rv.setProperty(PROP_GZIP, Boolean.toString(_gzip)); + rv.setProperty(PROP_MIN_GZIP_SIZE, Long.toString(_minGzipSize)); for (LogLimit lim : _limits) { rv.setProperty(PROP_RECORD_PREFIX + lim.getRootName(), Log.toLevelString(lim.getLimit())); @@ -812,16 +850,10 @@ public class LogManager implements Flushable { _consoleBuffer.clear(); } - private static final AtomicInteger __id = new AtomicInteger(); - private class ShutdownHook extends I2PAppThread { - private final int _id; - public ShutdownHook() { - _id = __id.incrementAndGet(); - } @Override public void run() { - setName("Log " + _id + " shutdown "); + setName("Log shutdown"); shutdown(); } }