From 1a385b6dca6051fd33aa158b3d4f05473cd9ddd2 Mon Sep 17 00:00:00 2001
From: zzz <zzz@mail.i2p>
Date: Fri, 18 Sep 2015 18:15:32 +0000
Subject: [PATCH] i2ptunnel: - Pass Accept-Encoding header through HTTP client
and server proxies, to allow end-to-end compression - Don't do
transparent response compression if response Content-Encoding indicates it
is already compressed - Minor encoding cleanups EepGet: - Send
Accept-Encoding: gzip even when proxied - Minor cleanups
---
.../i2ptunnel/HTTPResponseOutputStream.java | 24 ++++++++++++-------
.../i2p/i2ptunnel/I2PTunnelHTTPClient.java | 8 ++++---
.../i2p/i2ptunnel/I2PTunnelHTTPServer.java | 23 +++++++++++-------
core/java/src/net/i2p/util/EepGet.java | 15 ++++++------
4 files changed, 42 insertions(+), 28 deletions(-)
diff --git a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/HTTPResponseOutputStream.java b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/HTTPResponseOutputStream.java
index 83fbb6d0a6..2fc344a714 100644
--- a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/HTTPResponseOutputStream.java
+++ b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/HTTPResponseOutputStream.java
@@ -39,7 +39,10 @@ class HTTPResponseOutputStream extends FilterOutputStream {
private final byte _buf1[];
protected boolean _gzip;
protected long _dataExpected;
+ /** lower-case, trimmed */
protected String _contentType;
+ /** lower-case, trimmed */
+ protected String _contentEncoding;
private static final int CACHE_SIZE = 8*1024;
private static final ByteCache _cache = ByteCache.getInstance(8, CACHE_SIZE);
@@ -151,7 +154,7 @@ class HTTPResponseOutputStream extends FilterOutputStream {
responseLine = (responseLine.trim() + "\r\n");
if (_log.shouldLog(Log.INFO))
_log.info("Response: " + responseLine.trim());
- out.write(responseLine.getBytes());
+ out.write(DataHelper.getUTF8(responseLine));
} else {
for (int j = lastEnd+1; j < i; j++) {
if (_headerBuffer.getData()[j] == ':') {
@@ -160,7 +163,7 @@ class HTTPResponseOutputStream extends FilterOutputStream {
if ( (keyLen <= 0) || (valLen < 0) )
throw new IOException("Invalid header @ " + j);
String key = DataHelper.getUTF8(_headerBuffer.getData(), lastEnd+1, keyLen);
- String val = null;
+ String val;
if (valLen == 0)
val = "";
else
@@ -171,10 +174,10 @@ class HTTPResponseOutputStream extends FilterOutputStream {
String lcKey = key.toLowerCase(Locale.US);
if ("connection".equals(lcKey)) {
- out.write("Connection: close\r\n".getBytes());
+ out.write(DataHelper.getASCII("Connection: close\r\n"));
connectionSent = true;
} else if ("proxy-connection".equals(lcKey)) {
- out.write("Proxy-Connection: close\r\n".getBytes());
+ out.write(DataHelper.getASCII("Proxy-Connection: close\r\n"));
proxyConnectionSent = true;
} else if ("content-encoding".equals(lcKey) && "x-i2p-gzip".equals(val.toLowerCase(Locale.US))) {
_gzip = true;
@@ -189,7 +192,10 @@ class HTTPResponseOutputStream extends FilterOutputStream {
} catch (NumberFormatException nfe) {}
} else if ("content-type".equals(lcKey)) {
// save for compress decision on server side
- _contentType = val;
+ _contentType = val.toLowerCase(Locale.US);
+ } else if ("content-encoding".equals(lcKey)) {
+ // save for compress decision on server side
+ _contentEncoding = val.toLowerCase(Locale.US);
} else if ("set-cookie".equals(lcKey)) {
String lcVal = val.toLowerCase(Locale.US);
if (lcVal.contains("domain=b32.i2p") ||
@@ -203,7 +209,7 @@ class HTTPResponseOutputStream extends FilterOutputStream {
break;
}
}
- out.write((key.trim() + ": " + val.trim() + "\r\n").getBytes());
+ out.write(DataHelper.getUTF8(key.trim() + ": " + val + "\r\n"));
}
break;
}
@@ -214,9 +220,9 @@ class HTTPResponseOutputStream extends FilterOutputStream {
}
if (!connectionSent)
- out.write("Connection: close\r\n".getBytes());
+ out.write(DataHelper.getASCII("Connection: close\r\n"));
if (!proxyConnectionSent)
- out.write("Proxy-Connection: close\r\n".getBytes());
+ out.write(DataHelper.getASCII("Proxy-Connection: close\r\n"));
finishHeaders();
@@ -237,7 +243,7 @@ class HTTPResponseOutputStream extends FilterOutputStream {
protected boolean shouldCompress() { return _gzip; }
protected void finishHeaders() throws IOException {
- out.write("\r\n".getBytes()); // end of the headers
+ out.write(DataHelper.getASCII("\r\n")); // end of the headers
}
@Override
diff --git a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPClient.java b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPClient.java
index 74fcaccb1c..8645b686f4 100644
--- a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPClient.java
+++ b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPClient.java
@@ -881,7 +881,9 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn
} else if(lowercaseLine.startsWith("accept")) {
// strip the accept-blah headers, as they vary dramatically from
// browser to browser
- if(!Boolean.parseBoolean(getTunnel().getClientOptions().getProperty(PROP_ACCEPT))) {
+ // But allow Accept-Encoding: gzip, deflate
+ if(!lowercaseLine.startsWith("accept-encoding: ") &&
+ !Boolean.parseBoolean(getTunnel().getClientOptions().getProperty(PROP_ACCEPT))) {
line = null;
continue;
}
@@ -933,8 +935,8 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn
// according to rfc2616 s14.3, this *should* force identity, even if
// an explicit q=0 for gzip doesn't. tested against orion.i2p, and it
// seems to work.
- if(!Boolean.parseBoolean(getTunnel().getClientOptions().getProperty(PROP_ACCEPT)))
- newRequest.append("Accept-Encoding: \r\n");
+ //if (!Boolean.parseBoolean(getTunnel().getClientOptions().getProperty(PROP_ACCEPT)))
+ // newRequest.append("Accept-Encoding: \r\n");
if (!usingInternalOutproxy)
newRequest.append("X-Accept-Encoding: x-i2p-gzip;q=1.0, identity;q=0.5, deflate;q=0, gzip;q=0, *;q=0\r\n");
}
diff --git a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPServer.java b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPServer.java
index 6b3b417201..35dd6f1326 100644
--- a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPServer.java
+++ b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPServer.java
@@ -419,12 +419,14 @@ public class I2PTunnelHTTPServer extends I2PTunnelServer {
setEntry(headers, "Connection", "close");
// we keep the enc sent by the browser before clobbering it, since it may have
// been x-i2p-gzip
- String enc = getEntryOrNull(headers, "Accept-encoding");
- String altEnc = getEntryOrNull(headers, "X-Accept-encoding");
+ String enc = getEntryOrNull(headers, "Accept-Encoding");
+ String altEnc = getEntryOrNull(headers, "X-Accept-Encoding");
// according to rfc2616 s14.3, this *should* force identity, even if
// "identity;q=1, *;q=0" didn't.
- setEntry(headers, "Accept-encoding", "");
+ // as of 0.9.23, the client passes this header through, and we do the same,
+ // so if the server and browser can do the compression/decompression, we don't have to
+ //setEntry(headers, "Accept-Encoding", "");
socket.setReadTimeout(readTimeout);
Socket s = getSocket(socket.getPeerDestination().calculateHash(), socket.getLocalPort());
@@ -432,7 +434,7 @@ public class I2PTunnelHTTPServer extends I2PTunnelServer {
// instead of i2ptunnelrunner, use something that reads the HTTP
// request from the socket, modifies the headers, sends the request to the
// server, reads the response headers, rewriting to include Content-encoding: x-i2p-gzip
- // if it was one of the Accept-encoding: values, and gzip the payload
+ // if it was one of the Accept-Encoding: values, and gzip the payload
boolean allowGZIP = true;
String val = opts.getProperty("i2ptunnel.gzip");
if ( (val != null) && (!Boolean.parseBoolean(val)) )
@@ -443,7 +445,7 @@ public class I2PTunnelHTTPServer extends I2PTunnelServer {
boolean useGZIP = alt || ( (enc != null) && (enc.indexOf("x-i2p-gzip") >= 0) );
// Don't pass this on, outproxies should strip so I2P traffic isn't so obvious but they probably don't
if (alt)
- headers.remove("X-Accept-encoding");
+ headers.remove("X-Accept-Encoding");
String modifiedHeader = formatHeaders(headers, command);
if (_log.shouldLog(Log.DEBUG))
@@ -671,6 +673,7 @@ public class I2PTunnelHTTPServer extends I2PTunnelServer {
/**
* Don't compress small responses or images.
+ * Don't compress things that are already compressed.
* Compression is inline but decompression on the client side
* creates a new thread.
*/
@@ -687,7 +690,11 @@ public class I2PTunnelHTTPServer extends I2PTunnelServer {
(!_contentType.equals("application/x-bzip")) &&
(!_contentType.equals("application/x-bzip2")) &&
(!_contentType.equals("application/x-gzip")) &&
- (!_contentType.equals("application/zip"))));
+ (!_contentType.equals("application/zip")))) &&
+ (_contentEncoding == null ||
+ ((!_contentEncoding.equals("gzip")) &&
+ (!_contentEncoding.equals("compress")) &&
+ (!_contentEncoding.equals("deflate"))));
}
@Override
@@ -877,9 +884,9 @@ public class I2PTunnelHTTPServer extends I2PTunnelServer {
String lcName = name.toLowerCase(Locale.US);
if ("accept-encoding".equals(lcName))
- name = "Accept-encoding";
+ name = "Accept-Encoding";
else if ("x-accept-encoding".equals(lcName))
- name = "X-Accept-encoding";
+ name = "X-Accept-Encoding";
else if ("x-forwarded-for".equals(lcName))
name = "X-Forwarded-For";
else if ("x-forwarded-server".equals(lcName))
diff --git a/core/java/src/net/i2p/util/EepGet.java b/core/java/src/net/i2p/util/EepGet.java
index c31184dbdd..a098a997a3 100644
--- a/core/java/src/net/i2p/util/EepGet.java
+++ b/core/java/src/net/i2p/util/EepGet.java
@@ -755,6 +755,8 @@ public class EepGet {
Thread pusher = null;
_decompressException = null;
if (_isGzippedResponse) {
+ if (_log.shouldInfo())
+ _log.info("Gzipped response, starting decompressor");
PipedInputStream pi = BigPipedInputStream.getInstance();
PipedOutputStream po = new PipedOutputStream(pi);
pusher = new I2PAppThread(new Gunzipper(pi, _out), "EepGet Decompressor");
@@ -1160,17 +1162,13 @@ public class EepGet {
lookahead[1] = lookahead[2];
lookahead[2] = (byte)cur;
}
+
private static boolean isEndOfHeaders(byte lookahead[]) {
- byte first = lookahead[0];
- byte second = lookahead[1];
- byte third = lookahead[2];
- return (isNL(second) && isNL(third)) || // \n\n
- (isNL(first) && isNL(third)); // \n\r\n
+ return lookahead[2] == NL &&
+ (lookahead[0] == NL || lookahead[1] == NL); // \n\n or \n\r\n
}
- /** we ignore any potential \r, since we trim it on write anyway */
private static final byte NL = '\n';
- private static boolean isNL(byte b) { return (b == NL); }
/**
* @param timeout may be null
@@ -1315,7 +1313,8 @@ public class EepGet {
buf.append("Content-length: ").append(_postData.length()).append("\r\n");
// This will be replaced if we are going through I2PTunnelHTTPClient
buf.append("Accept-Encoding: ");
- if ((!_shouldProxy) &&
+ // as of 0.9.23, the proxy passes the Accept-Encoding header through
+ if ( /* (!_shouldProxy) && */
// This is kindof a hack, but if we are downloading a gzip file
// we don't want to transparently gunzip it and save it as a .gz file.
(!path.endsWith(".gz")) && (!path.endsWith(".tgz")))
--
GitLab