Util: Add https redirect support to EepGet and SSLEepGet

Add https support to EepGet CLI

core/java/src/net/i2p/util/EepGet.java

@@ -19,7 +19,6 @@ import java.net.URI;
 import java.net.URISyntaxException;
 import java.text.DecimalFormat;
 import java.util.ArrayList;
-import java.util.Date;
 import java.util.Formatter;
 import java.util.HashMap;
 import java.util.List;
@@ -184,6 +183,7 @@ public class EepGet {
     /**
      * EepGet [-p 127.0.0.1:4444] [-n #retries] [-e etag] [-o outputFile] [-m markSize lineLen] url
      *
+     * As of 0.9.45, supports https and redirect to https
      */
     public static void main(String args[]) {
         String proxyHost = "127.0.0.1";
@@ -295,7 +295,20 @@ public class EepGet {
         if (saveAs == null)
             saveAs = suggestName(url);
 
-        EepGet get = new EepGet(I2PAppContext.getGlobalContext(), true, proxyHost, proxyPort, numRetries, saveAs, url, true, etag);
+        EepGet get;
+        if (url.startsWith("https://")) {
+            if (etag != null) {
+                System.err.println("Etag option unsupported with https");
+                System.exit(1);
+            }
+            boolean shouldProxy = proxyHost != null && proxyHost.length() > 0 && proxyPort > 0;
+            if (shouldProxy)
+                get = new SSLEepGet(I2PAppContext.getGlobalContext(), SSLEepGet.ProxyType.HTTP, proxyHost, proxyPort, saveAs, url);
+            else
+                get = new SSLEepGet(I2PAppContext.getGlobalContext(), saveAs, url);
+        } else {
+            get = new EepGet(I2PAppContext.getGlobalContext(), true, proxyHost, proxyPort, numRetries, saveAs, url, true, etag);
+        }
         if (extra != null) {
             for (int i = 0; i < extra.size(); i += 2) {
                 get.addHeader(extra.get(i), extra.get(i + 1));
@@ -460,7 +473,6 @@ public class EepGet {
         private long _lastComplete;
         private boolean _firstTime;
         private final DecimalFormat _pct = new DecimalFormat("00.0%");
-        private final DecimalFormat _kbps = new DecimalFormat("###,000.00");
         public CLIStatusListener() { 
             this(1024, 40);
         }
@@ -530,7 +542,7 @@ public class EepGet {
             else
                 transferred = alreadyTransferred - _previousWritten;
             System.out.println();
-            System.out.println("== " + new Date());
+            //System.out.println("== " + new Date());
             if (notModified) {
                 System.out.println("== Source not modified since last download");
             } else {
@@ -543,27 +555,31 @@ public class EepGet {
                             + " bytes transferred" +
                             (_discarded > 0 ? (" and " + _discarded + " bytes discarded") : ""));
                 }
-                if (transferred > 0)
+                if (transferred > 0) {
-                    System.out.println("== Output saved to " + outputFile + " (" + alreadyTransferred + " bytes)");
+                    long sz = (new File(outputFile)).length();
+                    if (sz <= 0)
+                        sz = alreadyTransferred;
+                    System.out.println("== Output saved to " + outputFile + " (" + sz + " bytes)");
+                }
             }
             long timeToSend = _context.clock().now() - _startedOn;
             System.out.println("== Transfer time: " + DataHelper.formatDuration(timeToSend));
             if (_etag != null)
                 System.out.println("== ETag: " + _etag);            
             if (transferred > 0) {
-                StringBuilder buf = new StringBuilder(50);
+                StringBuilder buf = new StringBuilder(64);
                 buf.append("== Transfer rate: ");
-                double kbps = (1000.0d*(transferred)/(timeToSend*1024.0d));
+                if (timeToSend <= 0)
-                synchronized (_kbps) {
+                    timeToSend = 1;
-                    buf.append(_kbps.format(kbps));
+                long kbps = (long) (1000.0d * transferred / timeToSend);
-                }
+                buf.append(DataHelper.formatSize2Decimal(kbps, false));
-                buf.append("KBps");
+                buf.append("Bps");
                 System.out.println(buf.toString());
             }
         }
         public void attemptFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt, int numRetries, Exception cause) {
             System.out.println();
-            System.out.println("** " + new Date());
+            //System.out.println("** " + new Date());
             System.out.println("** Attempt " + currentAttempt + " of " + url + " failed");
             System.out.println("** Transfered " + bytesTransferred
                                + " with " + (bytesRemaining < 0 ? "unknown" : Long.toString(bytesRemaining)) + " remaining");
@@ -572,19 +588,19 @@ public class EepGet {
             _written = 0;
         }
         public void transferFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt) {
-            System.out.println("== " + new Date());
+            //System.out.println("== " + new Date());
             System.out.println("== Transfer of " + url + " failed after " + currentAttempt + " attempts");
             System.out.println("== Transfer size: " + bytesTransferred + " with "
                                + (bytesRemaining < 0 ? "unknown" : Long.toString(bytesRemaining)) + " remaining");
             long timeToSend = _context.clock().now() - _startedOn;
             System.out.println("== Transfer time: " + DataHelper.formatDuration(timeToSend));
-            double kbps = (timeToSend > 0 ? (1000.0d*(bytesTransferred)/(timeToSend*1024.0d)) : 0);
+            if (timeToSend <= 0)
-            StringBuilder buf = new StringBuilder(50);
+                timeToSend = 1;
+            long kbps = (long) (1000.0d * bytesTransferred / timeToSend);
+            StringBuilder buf = new StringBuilder(64);
             buf.append("== Transfer rate: ");
-            synchronized (_kbps) {
+            buf.append(DataHelper.formatSize2Decimal(kbps, false));
-                buf.append(_kbps.format(kbps));
+            buf.append("Bps");
-            }
-            buf.append("KBps");
             System.out.println(buf.toString());
         }
         public void attempting(String url) {}
@@ -738,7 +754,65 @@ public class EepGet {
                 if (_redirectLocation.startsWith("http://")) {
                     _actualURL = _redirectLocation;
                 } else if (_redirectLocation.startsWith("https://")) {
+                    // _proxy is the socket. It is null when extended by I2PSocketEepGet
+                    if (_proxy == null)
                         throw new IOException("Redirect to https unsupported");
+                    if (_postData != null)
+                        throw new IOException("Redirect to https unsupported");
+                    try { 
+                        _proxy.close(); 
+                        _proxy = null;
+                    } catch (IOException ioe) {}
+                    if (timeout != null)
+                        timeout.cancel();
+                    EepGet get;
+                    if (_shouldProxy) {
+                        if (_authState != null)
+                            throw new IOException("Redirect to https with proxy auth unsupported");
+                        if (_outputStream != null)
+                            get = new SSLEepGet(_context, SSLEepGet.ProxyType.HTTP, _proxyHost, _proxyPort, _outputStream, _redirectLocation);
+                        else
+                            get = new SSLEepGet(_context, SSLEepGet.ProxyType.HTTP, _proxyHost, _proxyPort, _outputFile, _redirectLocation);
+                    } else {
+                        if (_outputStream != null)
+                            get = new SSLEepGet(_context, _outputStream, _redirectLocation);
+                        else
+                            get = new SSLEepGet(_context, _outputFile, _redirectLocation);
+                    }
+                    if (_shouldWriteErrorToOutput)
+                        get.setWriteErrorToOutput();
+                    if (_extraHeaders != null) {
+                        for (String s : _extraHeaders) {
+                            String[] kv = DataHelper.split(s, ":", 2);
+                            if (kv.length == 2)
+                                get.addHeader(kv[0].trim(), kv[1].trim());
+                        }
+                    }
+                    synchronized (_listeners) {
+                        for (StatusListener sl : _listeners) {
+                            get.addStatusListener(sl);
+                        }
+                    }
+                    _actualURL = _redirectLocation;
+                    // reset some important variables, we don't want to save the values from the redirect
+                    _bytesRemaining = -1;
+                    _redirectLocation = null;
+                    _etag = _etagOrig;
+                    _lastModified = _lastModifiedOrig;
+                    _contentType = null;
+                    _encodingChunked = false;
+                    // TODO auth?
+                    // minSize/maxSize/maxRetries discarded
+                    _transferFailed = !get.fetch(_fetchHeaderTimeout, -1, _fetchInactivityTimeout);
+                    _keepFetching = false;
+                    // fixup the getters
+                    _responseCode = get.getStatusCode();
+                    _responseText = get.getStatusText();
+                    _contentType = get.getContentType();
+                    _etag = get.getETag();
+                    _lastModified = get.getLastModified();
+                    _notModified = get.getNotModified();
+                    return;
                 } else { 
                     // the Location: field has been required to be an absolute URI at least since
                     // RFC 1945 (HTTP/1.0 1996), so it isn't clear what the point of this is.
@@ -795,7 +869,7 @@ public class EepGet {
         }
         
         if (_log.shouldLog(Log.DEBUG))
-            _log.debug("Headers read completely, reading " + _bytesRemaining);
+            _log.debug("Headers read completely");
         
         boolean strictSize = (_bytesRemaining >= 0);
 
@@ -1174,8 +1248,8 @@ public class EepGet {
         String len = buf.toString().trim();
         try {
             long bytes = Long.parseLong(len, 16);
-            if (_log.shouldLog(Log.DEBUG))
+            //if (_log.shouldLog(Log.DEBUG))
-                _log.debug("Chunked length: " + bytes);
+            //    _log.debug("Chunked length: " + bytes);
             return bytes;
         } catch (NumberFormatException nfe) {
             throw new IOException("Invalid chunk length [" + len + "]");
@@ -1439,7 +1513,7 @@ public class EepGet {
         if (post)
             buf.append(_postData);
         if (_log.shouldLog(Log.DEBUG))
-            _log.debug("Request: [" + buf.toString() + "]");
+            _log.debug("Request:\n" + buf.toString().trim());
         return buf.toString();
     }
 

core/java/src/net/i2p/util/SSLEepGet.java

@@ -76,7 +76,7 @@ import net.i2p.socks.SOCKS5Client;
 
 /**
  * HTTPS only, no retries, no min and max size options, no timeout option
- * Fails on 301 or 302 (doesn't follow redirect)
+ * As of 0.9.45, supports redirect to https (redirect to http will still fail).
  * Fails on bad certs (must have a valid cert chain)
  * Self-signed certs or CAs not in the JVM key store must be loaded to be trusted.
  *
@@ -334,6 +334,9 @@ public class SSLEepGet extends EepGet {
                     proxyPort = 8080;
                 else
                     proxyPort = 1080;
+            } else if (proxyPort == 4444 && ptype != ProxyType.INTERNAL) {
+                if (proxyHost.equals("localhost") || proxyHost.equals("127.0.0.1") || proxyHost.equals("::1"))
+                    ptype = ProxyType.INTERNAL;
             }
             get = new SSLEepGet(I2PAppContext.getGlobalContext(), ptype, proxyHost, proxyPort, saveAs, url);
         } else {
@@ -567,7 +570,27 @@ public class SSLEepGet extends EepGet {
             _proxy.setSoTimeout(INACTIVITY_TIMEOUT);
 
         if (_redirectLocation != null) {
+            if (!_redirectLocation.startsWith("https://"))
                 throw new IOException("Server redirect to " + _redirectLocation + " not allowed");
+            _redirects++;
+            if (_redirects > 5)
+                throw new IOException("Too many redirects: to " + _redirectLocation);
+            if (_log.shouldInfo())
+                _log.info("Redirecting to " + _redirectLocation);
+            _actualURL = _redirectLocation;
+            AuthState as = _authState;
+            if (as != null)
+                as.authSent = false;
+            // reset some important variables, we don't want to save the values from the redirect
+            _bytesRemaining = -1;
+            _redirectLocation = null;
+            _etag = _etagOrig;
+            _lastModified = _lastModifiedOrig;
+            _contentType = null;
+            _encodingChunked = false;
+            sendRequest(timeout);
+            doFetch(timeout);
+            return;
         }
         
         if (_log.shouldLog(Log.DEBUG))