zzz/i2p.i2p
1
0
Fork 0
forked from I2P_Developers/i2p.i2p
Code Issues Pull Requests Packages Projects Releases Wiki Activity

2005-03-23 jrandom

Browse Source 
* New /configupdate.jsp page for controlling the update / notification
      process, as well as various minor related updates.  Note that not all
      options are exposed yet, and the update detection code isn't in place
      in this commit - it currently says there is always an update available.
    * New EepGet component for reliable downloading, with a CLI exposed in
      java -cp lib/i2p.jar net.i2p.util.EepGet url
    * Added a default signing key to the TrustedUpdate component to be used
      for verifying updates.  This signing key can be authenticated via
      gpg --verify i2p/core/java/src/net/i2p/crypto/TrustedUpdate.java
    * New public domain SHA1 implementation for the DSA code so that we can
      handle signing streams of arbitrary size without excess memory usage
      (thanks P.Verdy!)
    * Added some helpers to the TrustedUpdate to work off streams and to offer
      a minimal CLI:
          TrustedUpdate keygen pubKeyFile privKeyFile
          TrustedUpdate sign origFile signedFile privKeyFile
          TrustedUpdate verify signedFile
This commit is contained in:
jrandom
2005-03-23 21:13:03 +00:00
committed by zzz
parent 677eeac8f7
commit a2c309ddd3
18 changed files with 2149 additions and 201 deletions
Download Patch File Download Diff File Expand all files Collapse all files

513
core/java/src/net/i2p/util/EepGet.java Normal file
View File

@@ -0,0 +1,513 @@
package net.i2p.util;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.Socket;
import java.net.URL;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.StringTokenizer;
import java.util.Properties;
import net.i2p.I2PAppContext;
import net.i2p.data.DataHelper;
/**
* EepGet [-p localhost:4444]
* [-n #retries]
* [-o outputFile]
* [-m markSize lineLen]
* url
*/
public class EepGet {
private I2PAppContext _context;
private Log _log;
private boolean _shouldProxy;
private String _proxyHost;
private int _proxyPort;
private int _numRetries;
private String _outputFile;
private String _url;
private List _listeners;
private boolean _keepFetching;
private Socket _proxy;
private OutputStream _proxyOut;
private InputStream _proxyIn;
private OutputStream _out;
private long _alreadyTransferred;
private long _bytesTransferred;
private long _bytesRemaining;
private int _currentAttempt;
private String _etag;
public EepGet(I2PAppContext ctx, String proxyHost, int proxyPort, int numRetries, String outputFile, String url) {
this(ctx, true, proxyHost, proxyPort, numRetries, outputFile, url);
}
public EepGet(I2PAppContext ctx, int numRetries, String outputFile, String url) {
this(ctx, false, null, -1, numRetries, outputFile, url);
}
public EepGet(I2PAppContext ctx, boolean shouldProxy, String proxyHost, int proxyPort, int numRetries, String outputFile, String url) {
_context = ctx;
_log = ctx.logManager().getLog(EepGet.class);
_shouldProxy = shouldProxy;
_proxyHost = proxyHost;
_proxyPort = proxyPort;
_numRetries = numRetries;
_outputFile = outputFile;
_url = url;
_alreadyTransferred = 0;
_bytesTransferred = 0;
_bytesRemaining = -1;
_currentAttempt = 0;
_listeners = new ArrayList(1);
}
/**
* EepGet [-p localhost:4444] [-n #retries] [-o outputFile] [-m markSize lineLen] url
*
*/
public static void main(String args[]) {
String proxyHost = "localhost";
int proxyPort = 4444;
int numRetries = 5;
int markSize = 1024;
int lineLen = 40;
String saveAs = null;
String url = null;
try {
for (int i = 0; i < args.length; i++) {
if (args[i].equals("-p")) {
proxyHost = args[i+1].substring(0, args[i+1].indexOf(':'));
String port = args[i+1].substring(args[i+1].indexOf(':')+1);
proxyPort = Integer.parseInt(port);
i++;
} else if (args[i].equals("-n")) {
numRetries = Integer.parseInt(args[i+1]);
i++;
} else if (args[i].equals("-o")) {
saveAs = args[i+1];
i++;
} else if (args[i].equals("-m")) {
markSize = Integer.parseInt(args[i+1]);
lineLen = Integer.parseInt(args[i+2]);
i += 2;
} else {
url = args[i];
}
}
} catch (Exception e) {
e.printStackTrace();
usage();
return;
}
if (url == null) {
usage();
return;
}
if (saveAs == null)
saveAs = suggestName(url);
EepGet get = new EepGet(I2PAppContext.getGlobalContext(), proxyHost, proxyPort, numRetries, saveAs, url);
get.addStatusListener(get.new CLIStatusListener(markSize, lineLen));
get.fetch();
}
public static String suggestName(String url) {
String name = null;
if (url.lastIndexOf('/') >= 0)
name = sanitize(url.substring(url.lastIndexOf('/')+1));
if (name != null)
return name;
else
return sanitize(url);
}
private static final String _safeChars = "abcdefghijklmnopqrstuvwxyz" +
"ABCDEFGHIJKLMNOPQRSTUVWXYZ" +
"01234567890.,_=@#:";
private static String sanitize(String name) {
name = name.replace('/', '_');
StringBuffer buf = new StringBuffer(name);
for (int i = 0; i < name.length(); i++)
if (_safeChars.indexOf(buf.charAt(i)) == -1)
buf.setCharAt(i, '_');
return buf.toString();
}
private static void usage() {
System.err.println("EepGet [-p localhost:4444] [-n #retries] [-o outputFile] [-m markSize lineLen] url");
}
public static interface StatusListener {
public void bytesTransferred(long alreadyTransferred, int currentWrite, long bytesTransferred, long bytesRemaining, String url);
public void transferComplete(long alreadyTransferred, long bytesTransferred, long bytesRemaining, String url, String outputFile);
public void attemptFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt, int numRetries, Exception cause);
public void transferFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt);
}
private class CLIStatusListener implements StatusListener {
private int _markSize;
private int _lineSize;
private long _startedOn;
private long _written;
private long _lastComplete;
private DecimalFormat _pct = new DecimalFormat("00.0%");
private DecimalFormat _kbps = new DecimalFormat("###,000.00");
public CLIStatusListener() {
this(1024, 40);
}
public CLIStatusListener(int markSize, int lineSize) {
_markSize = markSize;
_lineSize = lineSize;
_written = 0;
_lastComplete = _context.clock().now();
_startedOn = _lastComplete;
}
public void bytesTransferred(long alreadyTransferred, int currentWrite, long bytesTransferred, long bytesRemaining, String url) {
for (int i = 0; i < currentWrite; i++) {
_written++;
if ( (_markSize > 0) && (_written % _markSize == 0) ) {
System.out.print("#");
if ( (_lineSize > 0) && (_written % ((long)_markSize*(long)_lineSize) == 0l) ) {
long now = _context.clock().now();
long timeToSend = now - _lastComplete;
if (timeToSend > 0) {
StringBuffer buf = new StringBuffer(50);
buf.append(" ");
double pct = ((double)alreadyTransferred + (double)_written) / ((double)alreadyTransferred + (double)bytesRemaining);
synchronized (_pct) {
buf.append(_pct.format(pct));
}
buf.append(": ");
buf.append(_written+alreadyTransferred);
buf.append(" @ ");
double lineKBytes = ((double)_markSize * (double)_lineSize)/1024.0d;
double kbps = lineKBytes/((double)timeToSend/1000.0d);
synchronized (_kbps) {
buf.append(_kbps.format(kbps));
}
buf.append("KBps");
buf.append(" / ");
long lifetime = _context.clock().now() - _startedOn;
double lifetimeKBps = (1000.0d*(double)(_written+alreadyTransferred)/((double)lifetime*1024.0d));
synchronized (_kbps) {
buf.append(_kbps.format(lifetimeKBps));
}
buf.append("KBps");
System.out.println(buf.toString());
}
_lastComplete = now;
}
}
}
}
public void transferComplete(long alreadyTransferred, long bytesTransferred, long bytesRemaining, String url, String outputFile) {
System.out.println();
System.out.println("== " + new Date());
System.out.println("== Transfer of " + url + " completed with " + (alreadyTransferred+bytesTransferred)
+ " and " + (bytesRemaining - bytesTransferred) + " remaining");
System.out.println("== Output saved to " + outputFile);
long timeToSend = _context.clock().now() - _startedOn;
System.out.println("== Transfer time: " + DataHelper.formatDuration(timeToSend));
StringBuffer buf = new StringBuffer(50);
buf.append("== Transfer rate: ");
double kbps = (1000.0d*(double)(_written)/((double)timeToSend*1024.0d));
synchronized (_kbps) {
buf.append(_kbps.format(kbps));
}
buf.append("KBps");
System.out.println(buf.toString());
}
public void attemptFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt, int numRetries, Exception cause) {
System.out.println();
System.out.println("** " + new Date());
System.out.println("** Attempt " + currentAttempt + " of " + url + " failed");
System.out.println("** Transfered " + bytesTransferred
+ " with " + (bytesRemaining < 0 ? "unknown" : ""+bytesRemaining) + " remaining");
System.out.println("** " + cause.getMessage());
_written = 0;
}
public void transferFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt) {
System.out.println("== " + new Date());
System.out.println("== Transfer of " + url + " failed after " + currentAttempt + " attempts");
System.out.println("== Transfer size: " + bytesTransferred + " with "
+ (bytesRemaining < 0 ? "unknown" : ""+bytesRemaining) + " remaining");
long timeToSend = _context.clock().now() - _startedOn;
System.out.println("== Transfer time: " + DataHelper.formatDuration(timeToSend));
double kbps = (timeToSend > 0 ? (1000.0d*(double)(bytesTransferred)/((double)timeToSend*1024.0d)) : 0);
StringBuffer buf = new StringBuffer(50);
buf.append("== Transfer rate: ");
synchronized (_kbps) {
buf.append(_kbps.format(kbps));
}
buf.append("KBps");
System.out.println(buf.toString());
}
}
public void addStatusListener(StatusListener lsnr) {
synchronized (_listeners) { _listeners.add(lsnr); }
}
public void stopFetching() { _keepFetching = false; }
public void fetch() {
_keepFetching = true;
if (_log.shouldLog(Log.DEBUG))
_log.debug("Fetching (proxied? " + _shouldProxy + ") url=" + _url);
while (_keepFetching) {
try {
sendRequest();
doFetch();
return;
} catch (IOException ioe) {
for (int i = 0; i < _listeners.size(); i++)
((StatusListener)_listeners.get(i)).attemptFailed(_url, _bytesTransferred, _bytesRemaining, _currentAttempt, _numRetries, ioe);
} finally {
if (_out != null) {
try {
_out.close();
} catch (IOException cioe) {}
_out = null;
}
if (_proxy != null) {
try {
_proxy.close();
_proxy = null;
} catch (IOException ioe) {}
}
}
_currentAttempt++;
if (_currentAttempt > _numRetries)
break;
try { Thread.sleep(5*1000); } catch (InterruptedException ie) {}
}
for (int i = 0; i < _listeners.size(); i++)
((StatusListener)_listeners.get(i)).transferFailed(_url, _bytesTransferred, _bytesRemaining, _currentAttempt);
}
/** return true if the URL was completely retrieved */
private void doFetch() throws IOException {
readHeaders();
if (_log.shouldLog(Log.DEBUG))
_log.debug("Headers read completely, reading " + _bytesRemaining);
byte buf[] = new byte[1024];
while (_keepFetching) {
int read = _proxyIn.read(buf);
if (read == -1)
break;
_out.write(buf, 0, read);
_bytesTransferred += read;
if (read > 0)
for (int i = 0; i < _listeners.size(); i++)
((StatusListener)_listeners.get(i)).bytesTransferred(_alreadyTransferred, read, _bytesTransferred, _bytesRemaining, _url);
}
if (_out != null)
_out.close();
_out = null;
if (_log.shouldLog(Log.DEBUG))
_log.debug("Done transferring " + _bytesTransferred);
if (_bytesRemaining == _bytesTransferred) {
for (int i = 0; i < _listeners.size(); i++)
((StatusListener)_listeners.get(i)).transferComplete(_alreadyTransferred, _bytesTransferred, _bytesRemaining, _url, _outputFile);
} else {
throw new IOException("Disconnection on attempt " + _currentAttempt + " after " + _bytesTransferred);
}
}
private void readHeaders() throws IOException {
String key = null;
StringBuffer buf = new StringBuffer(32);
boolean read = DataHelper.readLine(_proxyIn, buf);
if (!read) throw new IOException("Unable to read the first line");
int responseCode = handleStatus(buf.toString());
boolean rcOk = false;
switch (responseCode) {
case 200: // full
_out = new FileOutputStream(_outputFile, false);
rcOk = true;
break;
case 206: // partial
_out = new FileOutputStream(_outputFile, true);
rcOk = true;
break;
case 416: // completed (or range out of reach)
_bytesRemaining = 0;
_keepFetching = false;
return;
default:
rcOk = false;
}
byte lookahead[] = new byte[3];
while (true) {
int cur = _proxyIn.read();
switch (cur) {
case -1:
throw new IOException("Headers ended too soon");
case ':':
if (key == null) {
key = buf.toString();
buf.setLength(0);
increment(lookahead, cur);
break;
} else {
buf.append((char)cur);
increment(lookahead, cur);
break;
}
case '\n':
case '\r':
if (key != null)
handle(key, buf.toString());
buf.setLength(0);
key = null;
increment(lookahead, cur);
if (isEndOfHeaders(lookahead)) {
if (!rcOk)
throw new IOException("Invalid HTTP response code: " + responseCode);
return;
}
break;
default:
buf.append((char)cur);
increment(lookahead, cur);
}
if (buf.length() > 1024)
throw new IOException("Header line too long: " + buf.toString());
}
}
/**
* parse the first status line and grab the response code.
* e.g. "HTTP/1.1 206 OK" vs "HTTP/1.1 200 OK" vs
* "HTTP/1.1 404 NOT FOUND", etc.
*
* @return HTTP response code (200, 206, other)
*/
private int handleStatus(String line) {
if (_log.shouldLog(Log.DEBUG))
_log.debug("Status line: [" + line + "]");
StringTokenizer tok = new StringTokenizer(line, " ");
if (!tok.hasMoreTokens()) {
System.err.println("ERR: status "+ line);
return -1;
}
String protocol = tok.nextToken(); // ignored
if (!tok.hasMoreTokens()) {
System.err.println("ERR: status "+ line);
return -1;
}
String rc = tok.nextToken();
try {
return Integer.parseInt(rc);
} catch (NumberFormatException nfe) {
nfe.printStackTrace();
return -1;
}
}
private void handle(String key, String val) {
if (_log.shouldLog(Log.DEBUG))
_log.debug("Header line: [" + key + "] = [" + val + "]");
if (key.equalsIgnoreCase("Content-length")) {
try {
_bytesRemaining = Long.parseLong(val.trim());
} catch (NumberFormatException nfe) {
nfe.printStackTrace();
}
} else if (key.equalsIgnoreCase("ETag")) {
_etag = val.trim();
} else {
// ignore the rest
}
}
private void increment(byte[] lookahead, int cur) {
lookahead[0] = lookahead[1];
lookahead[1] = lookahead[2];
lookahead[2] = (byte)cur;
}
private boolean isEndOfHeaders(byte lookahead[]) {
byte first = lookahead[0];
byte second = lookahead[1];
byte third = lookahead[2];
return (isNL(second) && isNL(third)) || // \n\n
(isNL(first) && isNL(third)); // \n\r\n
}
/** we ignore any potential \r, since we trim it on write anyway */
private static final byte NL = '\n';
private boolean isNL(byte b) { return (b == NL); }
private void sendRequest() throws IOException {
File outFile = new File(_outputFile);
if (outFile.exists())
_alreadyTransferred = outFile.length();
String req = getRequest();
if (_shouldProxy) {
_proxy = new Socket(_proxyHost, _proxyPort);
} else {
try {
URL url = new URL(_url);
String host = url.getHost();
int port = url.getPort();
if (port == -1)
port = 80;
_proxy = new Socket(host, port);
} catch (MalformedURLException mue) {
throw new IOException("Request URL is invalid");
}
}
_proxyIn = _proxy.getInputStream();
_proxyOut = _proxy.getOutputStream();
_proxyOut.write(req.toString().getBytes());
_proxyOut.flush();
if (_log.shouldLog(Log.DEBUG))
_log.debug("Request flushed");
}
private String getRequest() {
StringBuffer buf = new StringBuffer(512);
buf.append("GET ").append(_url).append(" HTTP/1.1\n");
try {
URL url = new URL(_url);
buf.append("Host: ").append(url.getHost()).append("\n");
} catch (MalformedURLException mue) {
mue.printStackTrace();
}
if (_alreadyTransferred > 0) {
buf.append("Range: bytes=");
buf.append(_alreadyTransferred);
buf.append("-\n");
}
buf.append("Connection: close\n\n");
if (_log.shouldLog(Log.DEBUG))
_log.debug("Request: [" + buf.toString() + "]");
return buf.toString();
}
}

4
core/java/src/net/i2p/util/LogManager.java
View File

@@ -245,7 +245,7 @@ public class LogManager {
if (!_alreadyNoticedMissingConfig) {
if (_log.shouldLog(Log.WARN))
_log.warn("Log file " + _location + " does not exist");
System.err.println("Log file " + _location + " does not exist");
//System.err.println("Log file " + _location + " does not exist");
_alreadyNoticedMissingConfig = true;
}
parseConfig(new Properties());
@@ -644,7 +644,7 @@ public class LogManager {
}
public void shutdown() {
_log.log(Log.CRIT, "Shutting down logger");
_log.log(Log.WARN, "Shutting down logger");
_writer.flushRecords();
}