SusiMail: Use input streams for reading mail (ticket #2119)

Rewrite Base64, HeaderLine, and QuotedPrintable decoders
Rewrite and expansion of ReadBuffer class and utilities for streams
Rewrite Mail and MailPart to parse the headers only once
Rewrite MailPart parser
MailPart parser rewrite skips over the data without reading into memory or decoding
MailPart decoder rewrite to decode stream-to-stream
ReadBuffer becomes Buffer interface with multiple implementations
Logging and debugging tweaks

apps/susimail/src/src/i2p/susi/debug/Debug.java

@@ -44,11 +44,19 @@ public class Debug {
 		return level;
 	}
 
-	public static void debug( int msgLevel, String msg )
+	public static void debug( int msgLevel, String msg ) {
+		debug(msgLevel, msg, null);
+	}
+
+	/** @since 0.9.34 */
+	public static void debug(int msgLevel, String msg, Throwable t)
 	{
-		if( msgLevel <= level )
+		if( msgLevel <= level ) {
 			System.err.println("SusiMail: " + msg);
+			if (t != null)
+				t.printStackTrace();
+		}
 		if (msgLevel <= ERROR)
-			I2PAppContext.getGlobalContext().logManager().getLog(Debug.class).error(msg);
+			I2PAppContext.getGlobalContext().logManager().getLog(Debug.class).error(msg, t);
 	}
 }

apps/susimail/src/src/i2p/susi/util/Buffer.java

+package i2p.susi.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Base interface for all Buffers.
+ * Data may only be read or written via streams,
+ * unless implemented via additional methods in subclasses.
+ *
+ * @since 0.9.34
+ */
+public interface Buffer {
+
+	public InputStream getInputStream() throws IOException;
+
+	public OutputStream getOutputStream() throws IOException;
+
+	/**
+	 *  Top-level reader MUST call this to close the input stream.
+	 */
+	public void readComplete(boolean success);
+
+	/**
+	 *  Writer MUST call this when done.
+	 *  @param success if false, deletes any resources
+	 */
+	public void writeComplete(boolean success);
+
+	public int getLength();
+
+	public int getOffset();
+}

apps/susimail/src/src/i2p/susi/util/CountingInputStream.java

+package i2p.susi.util;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * An InputStream that implements ReadCounter.
+ *
+ * @since 0.9.34
+ */
+public class CountingInputStream extends FilterInputStream implements ReadCounter {
+
+    protected long count;
+    
+    /**
+     *
+     */
+    public CountingInputStream(InputStream in) {
+        super(in);
+    }
+    
+    @Override
+    public long skip(long n) throws IOException {
+	long rv = in.skip(n);
+        count += rv;
+        return rv;
+    }
+    
+    public long getRead() {
+        return count;
+    }
+
+    @Override
+    public int read() throws IOException {
+        int rv = in.read();
+        if (rv >= 0)
+            count++;
+        return rv;
+    }
+
+    @Override
+    public int read(byte buf[], int off, int len) throws IOException {
+        int rv = in.read(buf, off, len);
+        if (rv > 0)
+            count += rv;
+        return rv;
+    }
+    
+}

apps/susimail/src/src/i2p/susi/util/CountingOutputStream.java

+package i2p.susi.util;
+
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * An OutputStream that counts how many bytes are written
+ * and returns the count via getWritten().
+ *
+ * @since 0.9.34
+ */
+public class CountingOutputStream extends FilterOutputStream {
+    
+    private long count;
+
+    public CountingOutputStream(OutputStream out) {
+        super(out);
+    }
+    
+    public long getWritten() {
+        return count;
+    }
+
+    @Override
+    public void write(int val) throws IOException {
+        out.write(val);
+        count++;
+    }
+
+    @Override
+    public void write(byte src[], int off, int len) throws IOException {
+        out.write(src, off, len);
+        count += len;
+    }
+}

apps/susimail/src/src/i2p/susi/util/DecodingOutputStream.java

+package i2p.susi.util;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.Writer;
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CoderResult;
+
+
+/**
+ *  Buffering decoder, with output to a Writer.
+ *  Adapted from SAM UTF8Reader.
+ *
+ *  @since 0.9.34
+ */
+public class DecodingOutputStream extends OutputStream {
+
+    private final Writer _out;
+    private final ByteBuffer _bb;
+    private final CharBuffer _cb;
+    private final CharsetDecoder _dc;
+
+    // Charset.forName("UTF-8").newDecoder().replacement().charAt(0) & 0xffff
+    private static final int REPLACEMENT = 0xfffd;
+
+    /**
+     *  @param out UTF-8
+     */
+    public DecodingOutputStream(Writer out, String charset) {
+        super();
+        _out = out;
+        _dc = Charset.forName(charset).newDecoder();
+        _bb = ByteBuffer.allocate(1024);
+        _cb = CharBuffer.allocate(1024);
+    }						
+
+    @Override
+    public void write(int b) throws IOException {
+        if (!_bb.hasRemaining())
+            flush();
+        _bb.put((byte) b);
+    }
+
+    @Override
+    public void write(byte buf[], int off, int len) throws IOException {
+	while (len > 0) {
+            if (_bb.hasRemaining()) {
+	        int toWrite = Math.min(len, _bb.remaining());
+    	        _bb.put(buf, off, toWrite);
+                len -= toWrite;
+            }
+            flush();
+        }
+    }
+
+    private void decodeAndWrite(boolean endOfInput) throws IOException {
+        _bb.flip();
+	if (!_bb.hasRemaining())
+            return;
+        CoderResult result;
+        try {
+            result = _dc.decode(_bb, _cb, endOfInput);
+        } catch (IllegalStateException ise) {
+            System.out.println("Decoder error with endOfInput=" + endOfInput);
+            ise.printStackTrace();
+            result = null;
+        }
+        _bb.compact();
+        // Overflow and underflow are not errors.
+        // It seems to return underflow every time.
+        // So just check if we got a character back in the buffer.
+        if (result == null || (result.isError() && !_cb.hasRemaining())) {
+            _out.write(REPLACEMENT);
+        } else {
+            _cb.flip();
+            _out.append(_cb);
+            _cb.clear();
+        }
+    }
+
+    @Override
+    public void flush() throws IOException {
+        decodeAndWrite(false);
+    }
+
+    /** Only flushes. Does NOT close the writer */
+    @Override
+    public void close() throws IOException {
+        decodeAndWrite(true);
+    }
+
+/****
+    public static void main(String[] args) {
+        try {
+            String s = "Consider the encoding of the Euro sign, €." +
+                       " The Unicode code point for \"€\" is U+20AC.";
+            byte[] test = s.getBytes("UTF-8");
+            InputStream bais = new java.io.ByteArrayInputStream(test);
+            DecodingOutputStream r = new DecodingOutputStream(bais);
+            int b;
+            StringBuilder buf = new StringBuilder(128);
+            while ((b = r.write()) >= 0) {
+                buf.append((char) b);
+            }
+            System.out.println("Received: " + buf);
+            System.out.println("Test passed? " + buf.toString().equals(s));
+            buf.setLength(0);
+            bais = new java.io.ByteArrayInputStream(new byte[] { 'x', (byte) 0xcc, 'x' } );
+            r = new DecodingOutputStream(bais);
+            while ((b = r.write()) >= 0) {
+                buf.append((char) b);
+            }
+            System.out.println("Received: " + buf);
+        } catch (IOException ioe) {
+            ioe.printStackTrace();
+        }
+    }
+****/
+}

apps/susimail/src/src/i2p/susi/util/DummyOutputStream.java

+package i2p.susi.util;
+
+import java.io.OutputStream;
+
+/**
+ * Write to nowhere
+ *
+ * @since 0.9.34
+ */
+public class DummyOutputStream extends OutputStream {
+    
+    public DummyOutputStream() {
+        super();
+    }
+    
+    public void write(int val) {}
+
+    @Override
+    public void write(byte src[]) {}
+
+    @Override
+    public void write(byte src[], int off, int len) {}
+
+    @Override
+    public void flush() {}
+
+    @Override
+    public void close() {}
+}

apps/susimail/src/src/i2p/susi/util/EOFOnMatchInputStream.java

+package i2p.susi.util;
+
+import java.io.PushbackInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+
+import net.i2p.data.DataHelper;
+
+/**
+ *  A stream that returns EOF when the input matches
+ *  the bytes provided. The reader will never see any bytes
+ *  from a full match.
+ *
+ *  We extend PushbackInputStream for convenience,
+ *  but we use its buffer as a fifo, not a stack.
+ *  Do not call the unread() methods externally.
+ *
+ *  @since 0.9.34
+ */
+public class EOFOnMatchInputStream extends PushbackInputStream implements ReadCounter {
+    private final byte[] match;
+    private final int size;
+    private final ReadCounter cis;
+
+    /**
+     *  Non-counter mode. getRead() will return 0.
+     *  @param match will be copied
+     */
+    public EOFOnMatchInputStream(InputStream in, byte[] match) {
+        this(in, null, match);
+    }
+
+    /**
+     *  Counter mode. getRead() will the ReadCounter's value, not including the match bytes.
+     *  @param match will be copied
+     */
+    public EOFOnMatchInputStream(InputStream in, ReadCounter ctr, byte[] match) {
+        super(in, match.length);
+        size = match.length;
+        if (size <= 0)
+            throw new IllegalArgumentException();
+        // buf grows down, so flip for easy matching
+        this.match = reverse(match);
+        cis = ctr;
+    }
+
+    private static byte[] reverse(byte[] m) {
+        int j = m.length;
+        byte[] rv = new byte[j];
+        for (int i = 0; i < m.length; i++) {
+            rv[--j] = m[i];
+        }
+        return rv;
+    }
+
+    /**
+     *  If constructed with a counter, returns the count
+     *  (not necessarily starting at 0) minus the buffered/matched count.
+     *  Otherwise returns 0.
+     */
+    public long getRead() {
+        if (cis != null)
+        	return cis.getRead() - (size - pos);
+        return 0;
+    }
+
+    /**
+     *  @return true if we returned EOF because we hit the match
+     */
+    public boolean wasFound() {
+        return pos <= 0; 
+    }
+
+    /**
+     *  Debug only. Return the number of bytes currently in the buffer.
+     *
+     *  @return number of bytes buffered
+     */
+/*
+    public int getBuffered() {
+        return size - pos;
+    }
+*/
+
+    /**
+     *  Debug only. Return the buffer.
+     *
+     *  @return the buffer
+     */
+/*
+    public byte[] getBuffer() {
+        int len = getBuffered();
+        byte[] b = new byte[len];
+        if (len <= 0)
+            return b;
+        System.arraycopy(buf, pos, b, 0, len);
+        return reverse(b);
+    }
+*/
+
+    @Override
+    public int read() throws IOException {
+        if (pos <= 0) 
+            return -1;
+        while(true) {
+            // read, pushback, compare
+            int c = in.read();
+            if (c < 0) {
+                if (pos < size)
+                    return pop();
+                return -1;
+            }
+            if (pos >= size) {
+                // common case, buf is empty, no match
+                if (c != (match[size - 1] & 0xff))
+                    return c;
+                // push first byte into buf, go around again
+                unread(c);
+                continue;
+            }
+            unread(c);
+            if (!DataHelper.eq(buf, pos, match, pos, size - pos)) {
+                return pop();
+            }
+            // partial or full match
+            if (pos <= 0) 
+                return -1;   // full match
+            // partial match, go around again
+        }
+    }
+
+    /**
+     *  FIFO output. Pop the oldest (not the newest).
+     *  Only call if pos < size.
+     *  We never call super.read(), it returns the newest.
+     */
+    private int pop() {
+        // return oldest, shift up
+        int rv = buf[size - 1] & 0xff;
+        for (int i = size - 1; i > pos; i--) {
+            buf[i] = buf[i - 1];
+        }
+        pos++;
+        return rv;
+    }
+
+    @Override
+    public int read(byte buf[], int off, int len) throws IOException {
+        for (int i = 0; i < len; i++) {
+            int c = read();
+            if (c == -1) {
+                if (i == 0)
+                    return -1;
+                return i;
+            }
+            buf[off++] = (byte)c;
+        }
+        return len;
+    }
+
+    @Override
+    public long skip(long n) throws IOException {
+        long rv = 0;
+        int c;
+        while (rv < n && (c = read()) >= 0) {
+            rv++;
+        }
+        return rv;
+    }
+
+/****
+    public static void main(String[] args) {
+        String match = "xxa";
+        String test = "xxbxaxoaaxxyyyyyyxxxazzzz"; 
+        byte[] m = DataHelper.getASCII(match);
+        byte[] in = DataHelper.getASCII(test); 
+        try {
+            InputStream eof = new EOFOnMatchInputStream(new java.io.ByteArrayInputStream(in), m);
+            byte[] out = new byte[in.length + 10];
+            int read = eof.read(out);
+            if (read != test.indexOf(match))
+                System.out.println("EOFOMIS test failed, read " + read);
+            else if (!DataHelper.eq(in, 0, out, 0, read))
+                System.out.println("EOFOMIS test failed, bad data");
+            else
+                System.out.println("EOFOMIS test passed");
+        } catch (Exception e) {
+            System.out.println("EOFOMIS test failed");
+            e.printStackTrace();
+        }
+    }
+****/
+}    

apps/susimail/src/src/i2p/susi/util/EscapeHTMLOutputStream.java

+package i2p.susi.util;
+
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+import net.i2p.data.DataHelper;
+
+/**
+ * Escape HTML on the fly.
+ * Streaming version of DataHelper.escapeHTML(),
+ * and we escape '-' too since we stick debugging stuff inside comments,
+ * and '--' is disallowed inside comments.
+ *
+ * @since 0.9.34
+ */
+public class EscapeHTMLOutputStream extends FilterOutputStream {
+    
+    private static final byte[] AMP = DataHelper.getASCII("&");
+    private static final byte[] QUOT = DataHelper.getASCII(""");
+    private static final byte[] LT = DataHelper.getASCII("<");
+    private static final byte[] GT = DataHelper.getASCII(">");
+    private static final byte[] APOS = DataHelper.getASCII("'");
+    private static final byte[] MDASH = DataHelper.getASCII("-");
+    private static final byte[] BR = DataHelper.getASCII("<br>\n");
+
+
+    public EscapeHTMLOutputStream(OutputStream out) {
+        super(out);
+    }
+    
+    @Override
+    public void write(int val) throws IOException {
+        switch (val) {
+            case '&':
+                out.write(AMP);
+                break;
+            case '"':
+                out.write(QUOT);
+                break;
+            case '<':
+                out.write(LT);
+                break;
+            case '>':
+                out.write(GT);
+                break;
+            case '\'':
+                out.write(APOS);
+                break;
+            case '-':
+                out.write(MDASH);
+                break;
+            case '\r':
+                break;
+            case '\n':
+                out.write(BR);
+                break;
+            default:
+                out.write(val);
+        }
+    }
+
+    /**
+     *  Does nothing. Does not close the underlying stream.
+     */
+    @Override
+    public void close() {}
+}

apps/susimail/src/src/i2p/susi/util/EscapeHTMLWriter.java

+package i2p.susi.util;
+
+import java.io.FilterWriter;
+import java.io.IOException;
+import java.io.Writer;
+
+import net.i2p.data.DataHelper;
+
+/**
+ * Escape HTML on the fly.
+ * Streaming version of DataHelper.escapeHTML(),
+ * and we escape '-' too since we stick debugging stuff inside comments,
+ * and '--' is disallowed inside comments.
+ *
+ * @since 0.9.34
+ */
+public class EscapeHTMLWriter extends FilterWriter {
+    
+    private static final String AMP = "&";
+    private static final String QUOT = """;
+    private static final String LT = "<";
+    private static final String GT = ">";
+    private static final String APOS = "'";
+    private static final String MDASH = "-";
+    private static final String BR = "<br>\n";
+
+
+    public EscapeHTMLWriter(Writer out) {
+        super(out);
+    }
+    
+    @Override
+    public void write(int c) throws IOException {
+        switch (c) {
+            case '&':
+                out.write(AMP);
+                break;
+            case '"':
+                out.write(QUOT);
+                break;
+            case '<':
+                out.write(LT);
+                break;
+            case '>':
+                out.write(GT);
+                break;
+            case '\'':
+                out.write(APOS);
+                break;
+            case '-':
+                out.write(MDASH);
+                break;
+            case '\r':
+                break;
+            case '\n':
+                out.write(BR);
+                break;
+            default:
+                out.write(c);
+        }
+    }
+
+    @Override
+    public void write(char[] cbuf, int off, int len) throws IOException {
+        for (int i = off; i < off + len; i++) {
+            write(cbuf[i]);
+        }
+    }
+
+    @Override
+    public void write(String str, int off, int len) throws IOException {
+        for (int i = off; i < off + len; i++) {
+            write(str.charAt(i));
+        }
+    }
+
+    /**
+     *  Does nothing. Does not close the underlying writer.
+     */
+    @Override
+    public void close() {}
+}

apps/susimail/src/src/i2p/susi/util/FileBuffer.java

+package i2p.susi.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.OutputStream;
+
+import net.i2p.data.DataHelper;
+import net.i2p.util.SecureFileOutputStream;
+
+/**
+ * File implementation of Buffer.
+ *
+ * @since 0.9.34
+ */
+public class FileBuffer implements Buffer {
+
+	protected final File _file;
+	protected final int _offset;
+	protected final int _sublen;
+	private InputStream _is;
+	private OutputStream _os;
+	
+	public FileBuffer(File file) {
+		this(file, 0, 0);
+	}
+	
+	public FileBuffer(File file, int offset, int sublen) {
+		_file = file;
+		_offset = offset;
+		_sublen = sublen;
+	}
+
+	/**
+	 * @return the underlying file
+	 */
+	public File getFile() {
+		return _file;
+	}
+
+	/**
+         * Caller must call readComplete()
+         *
+	 * @return new FileInputStream
+	 */
+	public synchronized InputStream getInputStream() throws IOException {
+		if (_is != null && _offset <= 0)
+			return _is;
+		_is = new FileInputStream(_file);
+		if (_offset > 0)
+			DataHelper.skip(_is, _offset);
+		// TODO if _sublen > 0, wrap with a read limiter
+		return _is;
+	}
+
+	/**
+         * Caller must call writeComplete()
+         *
+	 * @return new FileOutputStream
+	 */
+	public synchronized OutputStream getOutputStream() throws IOException {
+		if (_os != null)
+			throw new IllegalStateException();
+		_os = new SecureFileOutputStream(_file);
+		return _os;
+	}
+
+	public synchronized void readComplete(boolean success) {
+		if (_is != null) {
+			try { _is.close(); } catch (IOException ioe) {}
+			_is = null;
+		}
+	}
+
+	/**
+	 * Deletes the file if success is false
+	 */
+	public synchronized void writeComplete(boolean success) {
+		if (_os != null) {
+			try { _os.close(); } catch (IOException ioe) {}
+			_os = null;
+		}
+		if (!success)
+			_file.delete();
+	}
+
+	/**
+	 * Always valid if file exists
+	 */
+	public int getLength() {
+		if (_sublen > 0)
+			return _sublen;
+		return (int) _file.length();
+	}
+
+	/**
+	 * Always valid
+	 */
+	public int getOffset() {
+		return _offset;
+	}
+
+	@Override
+	public String toString() {
+		return "FB " + _file;
+	}
+}

apps/susimail/src/src/i2p/susi/util/GzipFileBuffer.java

+package i2p.susi.util;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.OutputStream;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+import net.i2p.data.DataHelper;
+import net.i2p.util.SecureFileOutputStream;
+
+/**
+ * Gzip File implementation of Buffer.
+ *
+ * @since 0.9.34
+ */
+public class GzipFileBuffer extends FileBuffer {
+
+	private long _actualLength;
+	private CountingInputStream _cis;
+	private CountingOutputStream _cos;
+
+	public GzipFileBuffer(File file) {
+		super(file);
+	}
+	
+	public GzipFileBuffer(File file, int offset, int sublen) {
+		super(file, offset, sublen);
+	}
+
+	/**
+	 * @return new FileInputStream
+	 */
+	@Override
+	public synchronized InputStream getInputStream() throws IOException {
+		if (_cis != null && (_offset <= 0 || _offset == _cis.getRead()))
+			return _cis;
+		if (_cis != null && _offset > _cis.getRead()) {
+			DataHelper.skip(_cis, _offset - _cis.getRead());
+			return _cis;
+		}
+		_cis = new CountingInputStream(new GZIPInputStream(new BufferedInputStream(new FileInputStream(_file))));
+		if (_offset > 0)
+			DataHelper.skip(_cis, _offset);
+		// TODO if _sublen > 0, wrap with a read limiter
+		return _cis;
+	}
+
+	/**
+	 * @return new FileOutputStream
+	 */
+	@Override
+	public synchronized OutputStream getOutputStream() throws IOException {
+		if (_offset > 0)
+			throw new IllegalStateException();
+		if (_cos != null)
+			throw new IllegalStateException();
+		_cos = new CountingOutputStream(new BufferedOutputStream(new GZIPOutputStream(new SecureFileOutputStream(_file))));
+		return _cos;
+	}
+
+	@Override
+	public synchronized void readComplete(boolean success) {
+		if (_cis != null) {
+			if (success)
+				_actualLength = _cis.getRead();
+			try { _cis.close(); } catch (IOException ioe) {}
+			_cis = null;
+		}
+	}
+
+	/**
+	 * Sets the length if success is true
+	 */
+	@Override
+	public synchronized void writeComplete(boolean success) {
+		if (_cos != null) {
+			if (success)
+				_actualLength = _cos.getWritten();
+			try { _cos.close(); } catch (IOException ioe) {}
+			_cos = null;
+		}
+	}
+
+	/**
+         * Returns the actual uncompressed size.
+         *
+	 * Only known after reading and calling readComplete(true),
+	 * or after writing and calling writeComplete(true),
+	 * oherwise returns 0.
+	 */
+	@Override
+	public int getLength() {
+		return (int) _actualLength;
+	}
+
+	@Override
+	public String toString() {
+		return "GZFB " + _file;
+	}
+}

apps/susimail/src/src/i2p/susi/util/LimitInputStream.java

+package i2p.susi.util;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * Limit total reads and skips to a specified maximum, then return EOF
+ *
+ * @since 0.9.34
+ */
+public class LimitInputStream extends CountingInputStream {
+
+    private final long maxx;
+    
+    /**
+     *  @param max max number of bytes to read
+     */
+    public LimitInputStream(InputStream in, long max) {
+        super(in);
+        if (max < 0)
+            throw new IllegalArgumentException("negative limit: " + max);
+        maxx = max;
+    }
+    
+    @Override
+    public int available() throws IOException {
+        return (int) Math.min(maxx - count, super.available());
+    }
+    
+    @Override
+    public long skip(long n) throws IOException {
+        return super.skip(Math.min(maxx - count, n));
+    }
+    
+    @Override
+    public int read() throws IOException {
+        if (count >= maxx)
+            return -1;
+        return super.read();
+    }
+
+    @Override
+    public int read(byte buf[], int off, int len) throws IOException {
+        if (count >= maxx)
+            return -1;
+        return super.read(buf, off, (int) Math.min(maxx - count, len));
+    }
+    
+/****
+    public static void main(String[] args) {
+        try {
+            LimitInputStream lim = new LimitInputStream(new java.io.ByteArrayInputStream(new byte[20]), 5);
+            lim.read();
+            lim.skip(2);
+            byte[] out = new byte[10];
+            int read = lim.read(out);
+            if (read != 2)
+                System.out.println("LIS test failed, read " + read);
+            else if (lim.getRead() != 5)
+                System.out.println("CIS test failed, read " + lim.getRead());
+            else
+                System.out.println("LIS/CIS test passed");
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+****/
+}

apps/susimail/src/src/i2p/susi/util/MemoryBuffer.java

+package i2p.susi.util;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Buffer backed by a byte array.
+ * Use for small amounts of data only.
+ *
+ * @since 0.9.34
+ */
+public class MemoryBuffer implements Buffer {
+
+	private ByteArrayOutputStream _baos;
+	private byte content[];
+	private final int _size;
+	
+	public MemoryBuffer() {
+		this(4096);
+	}
+
+	public MemoryBuffer(int size) {
+		_size = size;
+	}
+
+	/**
+	 * @return new ByteArrayInputStream
+	 */
+	public synchronized InputStream getInputStream() throws IOException {
+		if (content == null)
+			throw new IOException("no data");
+		return new ByteArrayInputStream(content);
+	}
+
+	/**
+	 * @return new or existing ByteArrayOutputStream
+	 */
+	public synchronized OutputStream getOutputStream() {
+		if (_baos == null)
+			_baos = new ByteArrayOutputStream(_size);
+		return _baos;
+	}
+
+	public void readComplete(boolean success) {}
+
+	/**
+	 * Deletes the data if success is false
+	 */
+	public synchronized void writeComplete(boolean success) {
+		if (success) {
+			if (content == null)
+				content = _baos.toByteArray();
+		} else {
+			content = null;
+		}
+		_baos = null;
+	}
+
+	/**
+	 * Current size.
+	 */
+	public synchronized int getLength() {
+		if (content != null)
+			return content.length;
+		if (_baos != null)
+			return _baos.size();
+		return 0;
+	}
+
+	/**
+	 * @return 0 always
+	 */
+	public int getOffset() {
+		return 0;
+	}
+
+	/**
+	 * @return content if writeComplete(true) was called, otherwise null
+	 */
+	public byte[] getContent() {
+		return content;
+	}
+
+	@Override
+	public String toString() {
+		return "SB " + (content == null ? "empty" : content.length + " bytes");
+	}
+}

apps/susimail/src/src/i2p/susi/util/OutputStreamBuffer.java

+package i2p.susi.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Output only. Input unsupported.
+ *
+ * @since 0.9.34
+ */
+public class OutputStreamBuffer implements Buffer {
+
+	private final OutputStream _out;
+
+	public OutputStreamBuffer(OutputStream out) {
+		_out = out;
+	}
+
+	/**
+	 * @throws UnsupportedOperationException
+	 */
+	public InputStream getInputStream() {
+		throw new UnsupportedOperationException();
+	}
+
+	/**
+	 * @return new OutputStreamOutputStream
+	 */
+	public OutputStream getOutputStream() {
+		return _out;
+	}
+
+	/**
+	 * Does nothing
+	 */
+	public void readComplete(boolean success) {}
+
+	/**
+	 * Closes the output stream
+	 */
+	public void writeComplete(boolean success) {
+		try { _out.close(); } catch (IOException ioe) {}
+	}
+
+	/**
+	 * @return 0 always
+	 */
+	public int getLength() {
+		return 0;
+	}
+
+	/**
+	 * @return 0 always
+	 */
+	public int getOffset() {
+		return 0;
+	}
+
+	@Override
+	public String toString() {
+		return "OSB";
+	}
+}

apps/susimail/src/src/i2p/susi/util/ReadBuffer.java

@@ -23,12 +23,19 @@
  */
 package i2p.susi.util;
 
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+
 import net.i2p.data.DataHelper;
 
 /**
+ * Input only for constant data, initialized from a byte array.
+ * See MemoryBuffer for read/write.
+ *
  * @author susi
  */
-public class ReadBuffer {
+public class ReadBuffer implements Buffer {
 
 	public final byte content[];
 	public final int length, offset;
@@ -39,6 +46,49 @@ public class ReadBuffer {
 		this.length = length;
 	}
 
+	/**
+	 * @return new ByteArrayInputStream over the content
+	 * @since 0.9.34
+	 */
+	public InputStream getInputStream() {
+		return new ByteArrayInputStream(content, offset, length);
+	}
+
+	/**
+	 * @throws IllegalStateException always
+	 * @since 0.9.34
+	 */
+	public OutputStream getOutputStream() {
+		throw new IllegalStateException();
+	}
+
+	/**
+	 * Does nothing
+	 * @since 0.9.34
+	 */
+	public void readComplete(boolean success) {}
+
+	/**
+	 * Does nothing
+	 * @since 0.9.34
+	 */
+	public void writeComplete(boolean success) {}
+
+	/**
+	 * Always valid
+	 */
+	public int getLength() {
+		return length;
+	}
+
+	/**
+	 * Always valid
+	 */
+	public int getOffset() {
+		return offset;
+	}
+
+	@Override
 	public String toString()
 	{
 		return content != null ? DataHelper.getUTF8(content, offset, length) : "";

apps/susimail/src/src/i2p/susi/util/ReadCounter.java

+package i2p.susi.util;
+
+/**
+ * Count the bytes that have been read or skipped
+ *
+ * @since 0.9.34
+ */
+public interface ReadCounter {
+
+    /**
+     *  The total number of bytes that have been read or skipped
+     */
+    public long getRead();
+}

apps/susimail/src/src/i2p/susi/webmail/Mail.java

@@ -23,15 +23,19 @@
  */
 package i2p.susi.webmail;
 
-import i2p.susi.util.Config;
 import i2p.susi.debug.Debug;
-import i2p.susi.util.ReadBuffer;
-import i2p.susi.webmail.encoding.DecodingException;
+import i2p.susi.util.Buffer;
+import i2p.susi.util.Config;
+import i2p.susi.util.CountingInputStream;
+import i2p.susi.util.EOFOnMatchInputStream;
+import i2p.susi.util.MemoryBuffer;
 import i2p.susi.webmail.encoding.Encoding;
 import i2p.susi.webmail.encoding.EncodingFactory;
 
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.text.DateFormat;
 import java.text.ParseException;
@@ -59,6 +63,11 @@ class Mail {
 	private static final String P2 = "^<[^@< \t]+@[^> \t]+>$";
 	private static final Pattern PATTERN1 = Pattern.compile(P1);
 	private static final Pattern PATTERN2 = Pattern.compile(P2);
+	/**
+	 *  Also used by MailPart
+	 *  See MailPart for why we don't do \r\n\r\n
+	 */
+	static final byte HEADER_MATCH[] = DataHelper.getASCII("\r\n\r");
 
 	private int size;
 	public String sender,   // as received, trimmed only, not HTML escaped
@@ -72,7 +81,7 @@ class Mail {
 		quotedDate;  // Current Locale, local time zone, longer format
 	public final String uidl;
 	public Date date;
-	private ReadBuffer header, body;
+	private Buffer header, body;
 	private MailPart part;
 	String[] to, cc;        // addresses only, enclosed by <>
 	private boolean isNew, isSpam;
@@ -100,15 +109,27 @@ class Mail {
 	 *  This may or may not contain the body also.
 	 *  @return if null, nothing has been loaded yet for this UIDL
 	 */
-	public synchronized ReadBuffer getHeader() {
+	public synchronized Buffer getHeader() {
 		return header;
 	}
 
-	public synchronized void setHeader(ReadBuffer rb) {
+	public synchronized void setHeader(Buffer rb) {
+		try {
+			setHeader(rb, rb.getInputStream(), true);
+		} catch (IOException ioe) {
+			// TODO...
+		}
+	}
+
+	/** @since 0.9.34 */
+	private synchronized String[] setHeader(Buffer rb, InputStream in, boolean closeIn) {
 		if (rb == null)
-			return;
+			return null;
 		header = rb;
-		parseHeaders();
+		String[] rv = parseHeaders(in);
+		if (closeIn)
+			rb.readComplete(true);
+		return rv;
 	}
 
 	/**
@@ -122,23 +143,40 @@ class Mail {
          *  This contains the header also.
          *  @return may be null
          */
-	public synchronized ReadBuffer getBody() {
+	public synchronized Buffer getBody() {
 		return body;
 	}
 
-	public synchronized void setBody(ReadBuffer rb) {
+	public synchronized void setBody(Buffer rb) {
 		if (rb == null)
 			return;
-		if (header == null)
-			setHeader(rb);
+		// In the common case where we have the body, we only parse the headers once.
+		// we always re-set the header, even if it was non-null before,
+		// as we have to parse them to find the start of the body
+		// and who knows, the headers could have changed.
+		//if (header == null)
+		//	setHeader(rb);
 		body = rb;
-		size = rb.length;
+		boolean success = false;
+		CountingInputStream in = null;
 		try {
-			part = new MailPart(uidl, rb);
-		} catch (DecodingException de) {
-			Debug.debug(Debug.ERROR, "Decode error: " + de);
+			in = new CountingInputStream(rb.getInputStream());
+			String[] headerLines = setHeader(rb, in, false);
+			// TODO just fail?
+			if (headerLines == null)
+				headerLines = new String[0];
+			part = new MailPart(uidl, rb, in, in, headerLines);
+			rb.readComplete(true);
+			// may only be available after reading and calling readComplete()
+			size = rb.getLength();
+			success = true;
+		} catch (IOException de) {
+			Debug.debug(Debug.ERROR, "Decode error", de);
 		} catch (RuntimeException e) {
-			Debug.debug(Debug.ERROR, "Parse error: " + e);
+			Debug.debug(Debug.ERROR, "Parse error", e);
+		} finally {
+			try { in.close(); } catch (IOException ioe) {}
+			rb.readComplete(success);
 		}
 	}
 
@@ -293,8 +331,12 @@ class Mail {
 		longLocalDateFormatter.setTimeZone(tz);
 	}
 
-	private void parseHeaders()
+	/**
+	 * @return all headers, to pass to MailPart, or null on error
+	 */
+	private String[] parseHeaders(InputStream in)
 	{
+		String[] headerLines = null;
 		error = "";
 		if( header != null ) {
 
@@ -317,10 +359,15 @@ class Mail {
 			if( ok ) {
 				
 				try {
-					ReadBuffer decoded = hl.decode( header );
-					BufferedReader reader = new BufferedReader( new InputStreamReader( new ByteArrayInputStream( decoded.content, decoded.offset, decoded.length ), "UTF-8" ) );
-					String line;
-					while( ( line = reader.readLine() ) != null ) {
+					EOFOnMatchInputStream eofin = new EOFOnMatchInputStream(in, HEADER_MATCH);
+					MemoryBuffer decoded = new MemoryBuffer(4096);
+					hl.decode(eofin, decoded);
+					if (!eofin.wasFound())
+						Debug.debug(Debug.DEBUG, "EOF hit before \\r\\n\\r\\n in Mail");
+					// Fixme UTF-8 to bytes to UTF-8
+					headerLines = DataHelper.split(new String(decoded.getContent(), decoded.getOffset(), decoded.getLength()), "\r\n");
+					for (int j = 0; j < headerLines.length; j++) {
+						String line = headerLines[j];
 						if( line.length() == 0 )
 							break;
 
@@ -418,9 +465,11 @@ class Mail {
 				}
 				catch( Exception e ) {
 					error += "Error parsing mail header: " + e.getClass().getName() + '\n';
+					Debug.debug(Debug.ERROR, "Parse error", e);
 				}		
 			}
 		}
+		return headerLines;
 	}
 }
 

apps/susimail/src/src/i2p/susi/webmail/MailCache.java

@@ -25,10 +25,14 @@ package i2p.susi.webmail;
 
 import i2p.susi.debug.Debug;
 import i2p.susi.util.Config;
+import i2p.susi.util.Buffer;
+import i2p.susi.util.FileBuffer;
 import i2p.susi.util.ReadBuffer;
+import i2p.susi.util.MemoryBuffer;
 import i2p.susi.webmail.pop3.POP3MailBox;
 import i2p.susi.webmail.pop3.POP3MailBox.FetchRequest;
 
+import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -38,6 +42,8 @@ import java.util.Hashtable;
 import java.util.List;
 import java.util.Set;
 
+import net.i2p.I2PAppContext;
+
 /**
  * @author user
  */
@@ -50,16 +56,17 @@ class MailCache {
 	private final POP3MailBox mailbox;
 	private final Hashtable<String, Mail> mails;
 	private final PersistentMailCache disk;
+	private final I2PAppContext _context;
 	
 	/** Includes header, headers are generally 1KB to 1.5 KB,
 	 *  and bodies will compress well.
          */
-	private static final int FETCH_ALL_SIZE = 8192;
+	private static final int FETCH_ALL_SIZE = 32*1024;
 
 	/**
 	 * @param mailbox non-null
 	 */
-	MailCache(POP3MailBox mailbox, 
+	MailCache(I2PAppContext ctx, POP3MailBox mailbox,
 		  String host, int port, String user, String pass) {
 		this.mailbox = mailbox;
 		mails = new Hashtable<String, Mail>();
@@ -71,6 +78,7 @@ class MailCache {
 			Debug.debug(Debug.ERROR, "Error creating disk cache: " + ioe);
 		}
 		disk = pmc;
+		_context = ctx;
 		if (disk != null)
 			loadFromDisk();
 	}
@@ -142,7 +150,8 @@ class MailCache {
 				mail.setHeader(mailbox.getHeader(uidl));
 		} else if (mode == FetchMode.ALL) {
 			if(!mail.hasBody()) {
-				ReadBuffer rb = mailbox.getBody(uidl);
+				File file = new File(_context.getTempDir(), "susimail-new-" + _context.random().nextLong());
+				Buffer rb = mailbox.getBody(uidl, new FileBuffer(file));
 				if (rb != null) {
 					mail.setBody(rb);
 					if (disk != null && disk.saveMail(mail) &&
@@ -216,7 +225,7 @@ class MailCache {
 							continue;  // found on disk, woo
 						}
 					}
-					POP3Request pr = new POP3Request(mail, true);
+					POP3Request pr = new POP3Request(mail, true, new MemoryBuffer(1024));
 					fetches.add(pr);
 				} else {
 					if (mail.hasBody() &&
@@ -238,7 +247,8 @@ class MailCache {
 							continue;  // found on disk, woo
 						}
 					}
-					POP3Request pr = new POP3Request(mail, false);
+					File file = new File(_context.getTempDir(), "susimail-new-" + _context.random().nextLong());
+					POP3Request pr = new POP3Request(mail, false, new FileBuffer(file));
 					fetches.add(pr);
 				} else {
 					if (!Boolean.parseBoolean(Config.getProperty(WebMail.CONFIG_LEAVE_ON_SERVER))) {
@@ -258,15 +268,14 @@ class MailCache {
 			mailbox.getBodies(bar);
 			//  Process results
 			for (POP3Request pr : fetches) {
-				ReadBuffer rb = pr.buf;
-				if (rb != null) {
+				if (pr.getSuccess()) {
 					Mail mail = pr.mail;
 					if (!mail.hasHeader())
 						mail.setNew(true);
 					if (pr.getHeaderOnly()) {
-						mail.setHeader(rb);
+						mail.setHeader(pr.getBuffer());
 					} else {
-						mail.setBody(rb);
+						mail.setBody(pr.getBuffer());
 					}
 					rv = true;
 					if (disk != null) {
@@ -326,24 +335,41 @@ class MailCache {
 	 */
 	private static class POP3Request implements FetchRequest {
 		public final Mail mail;
-		private final boolean headerOnly;
-		public ReadBuffer buf;
+		private boolean headerOnly, success;
+		public final Buffer buf;
 
-		public POP3Request(Mail m, boolean hOnly) {
+		public POP3Request(Mail m, boolean hOnly, Buffer buffer) {
 			mail = m;
 			headerOnly = hOnly;
+			buf = buffer;
 		}
 
 		public String getUIDL() {
 			return mail.uidl;
 		}
 
-		public boolean getHeaderOnly() {
+		/** @since 0.9.34 */
+		public synchronized void setHeaderOnly(boolean headerOnly) {
+			this.headerOnly = headerOnly;
+		}
+
+		public synchronized boolean getHeaderOnly() {
 			return headerOnly;
 		}
 
-		public void setBuffer(ReadBuffer buffer) {
-			buf = buffer;
+		/** @since 0.9.34 */
+		public Buffer getBuffer() {
+			return buf;
+		}
+
+		/** @since 0.9.34 */
+		public synchronized void setSuccess(boolean success) {
+			this.success = success;
+		}
+
+		/** @since 0.9.34 */
+		public synchronized boolean getSuccess() {
+			return success;
 		}
 	}
 }

apps/susimail/src/src/i2p/susi/webmail/MailPart.java

@@ -24,11 +24,22 @@
 package i2p.susi.webmail;
 
 import i2p.susi.debug.Debug;
+import i2p.susi.util.Buffer;
+import i2p.susi.util.CountingOutputStream;
+import i2p.susi.util.DummyOutputStream;
+import i2p.susi.util.EOFOnMatchInputStream;
+import i2p.susi.util.LimitInputStream;
 import i2p.susi.util.ReadBuffer;
+import i2p.susi.util.ReadCounter;
+import i2p.susi.util.OutputStreamBuffer;
+import i2p.susi.util.MemoryBuffer;
 import i2p.susi.webmail.encoding.DecodingException;
 import i2p.susi.webmail.encoding.Encoding;
 import i2p.susi.webmail.encoding.EncodingFactory;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
@@ -40,53 +51,68 @@ import net.i2p.data.DataHelper;
  */
 class MailPart {
 
+	private static final OutputStream DUMMY_OUTPUT = new DummyOutputStream();
 	public final String[] headerLines;
 	public final String type, encoding, name,
 		description, disposition, charset, version;
+	/** begin, end, and beginBody are relative to readBuffer.getOffset().
+         *  begin is before the headers
+         *  beginBody is after the headers
+	 *  warning - end is exclusive
+         */
 	private final int beginBody, begin, end;
 	/** fixme never set */
 	public final String filename = null;
 	public final List<MailPart> parts;
 	public final boolean multipart, message;
-	public final ReadBuffer buffer;
+	public final Buffer buffer;
+
+	/**
+	 *  the decoded length if known, else -1
+	 *  @since 0.9.34
+	 */
+	public int decodedLength = -1;
+
 	/**
 	 *  the UIDL of the mail, same for all parts
 	 *  @since 0.9.33
 	 */
 	public final String uidl;
 	
-
-	public MailPart(String uidl,  ReadBuffer readBuffer) throws DecodingException
-	{
-		this(uidl, readBuffer, readBuffer.offset, readBuffer.length);
-	}
-
-	public MailPart(String uidl, ReadBuffer readBuffer, int offset, int length) throws DecodingException
+	/**
+	 *  @param readBuffer has zero offset for top-level MailPart.
+	 *  @param in used for reading (NOT readBuffer.getInputStream())
+	 *  @param counter used for counting how much we have read.
+         *                 Probably the same as InputStream but a different interface.
+	 *  @param hdrlines non-null for top-level MailPart, where they
+	 *         were already parsed in Mail. Null otherwise
+	 */
+	public MailPart(String uidl, Buffer readBuffer, InputStream in, ReadCounter counter, String[] hdrlines) throws IOException
 	{
 		this.uidl = uidl;
-		begin = offset;
-		end = offset + length;
 		buffer = readBuffer;
 		
 		parts = new ArrayList<MailPart>();
 
-		/*
-		 * parse header lines
-		 */
-		int bb = end;
-		for( int i = begin; i < end - 4; i++ ) {
-			if( buffer.content[i] == '\r' &&
-					buffer.content[i+1] == '\n' &&
-					buffer.content[i+2] == '\r' &&
-					buffer.content[i+3] == '\n' ) {
-				bb = i + 2;
-				break;
-			}
+		if (hdrlines != null) {
+			// from Mail headers
+			headerLines = hdrlines;
+			begin = 0;
+		} else {
+			begin = (int) counter.getRead();
+			// parse header lines
+			// We don't do \r\n\r\n because then we can miss the \r\n--
+			// of the multipart boundary. So we do \r\n\r here,
+			// and \n-- below. If it's not multipart, we will swallow the
+			// \n below.
+			EOFOnMatchInputStream eofin = new EOFOnMatchInputStream(in, Mail.HEADER_MATCH);
+			MemoryBuffer decodedHeaders = new MemoryBuffer(4096);
+			EncodingFactory.getEncoding("HEADERLINE").decode(eofin, decodedHeaders);
+			if (!eofin.wasFound())
+				Debug.debug(Debug.DEBUG, "EOF hit before \\r\\n\\r\\n in MailPart");
+			// Fixme UTF-8 to bytes to UTF-8
+			headerLines = DataHelper.split(new String(decodedHeaders.getContent(), decodedHeaders.getOffset(), decodedHeaders.getLength()), "\r\n");
 		}
-		beginBody = bb;
-			
-		ReadBuffer decodedHeaders = EncodingFactory.getEncoding( "HEADERLINE" ).decode( buffer.content, begin, beginBody - begin );
-		headerLines = DataHelper.split(new String(decodedHeaders.content, decodedHeaders.offset, decodedHeaders.length), "\r\n");
 
 		String boundary = null;
 		String x_encoding = null;
@@ -123,7 +149,7 @@ class MailPart {
 					boundary = str;
 				if (x_type.startsWith( "multipart" ) && boundary != null )
 					x_multipart = true;
-				if (x_type.startsWith( "message" ) )
+				else if (x_type.startsWith( "message" ) )
 					x_message = true;
 				str = getHeaderLineAttribute( headerLines[i], "name" );
 				if( str != null )
@@ -150,60 +176,117 @@ class MailPart {
 		description = x_description;
 		version = x_version;
 
+		// see above re: \n
+		if (multipart) {
+			// EOFOnMatch will eat the \n
+			beginBody = (int) counter.getRead() + 1;
+		} else {
+			// swallow the \n
+			int c = in.read();
+			if (c != '\n')
+				Debug.debug(Debug.DEBUG, "wasn't a \\n, it was " + c);
+			beginBody = (int) counter.getRead();
+		}
+
+		int tmpEnd = 0;
 		/*
 		 * parse body
 		 */
-		int beginLastPart = -1;
 		if( multipart ) {
-			byte boundaryArray[] = DataHelper.getUTF8(boundary);
-			for( int i = beginBody; i < end - 4; i++ ) {
-				if( buffer.content[i] == '\r' &&
-						buffer.content[i+1] == '\n' &&
-						buffer.content[i+2] == '-' &&
-						buffer.content[i+3] == '-' ) {
-					/*
-					 * begin of possible boundary line
-					 */
-					int j = 0;
-					for( ; j < boundaryArray.length && i + 4 + j < end; j++ )
-						if( buffer.content[ i + 4 + j ] != boundaryArray[j] )
-							break;
-					if( j == boundaryArray.length ) {
-						int k = i + 4 + j;
-						if( k < end - 2 &&
-								buffer.content[k] == '-' &&
-								buffer.content[k+1] == '-' )
-							k += 2;
-						
-						if( k < end - 2 &&
-								buffer.content[k] == '\r' &&
-								buffer.content[k+1] == '\n' ) {
-							
-							k += 2;
-							
-							if( beginLastPart != -1 ) {
-								int endLastPart = Math.min(i + 2, end);
-								MailPart newPart = new MailPart(uidl, buffer, beginLastPart, endLastPart - beginLastPart);
-								parts.add( newPart );
-							}
-							beginLastPart = k;
-						}
-						i = k;
+			// See above for why we don't include the \r
+			byte[] match = DataHelper.getASCII("\n--" + boundary);
+			for (int i = 0; ; i++) {
+				EOFOnMatchInputStream eofin = new EOFOnMatchInputStream(in, counter, match);
+				if (i == 0) {
+					// Read through first boundary line, not including "\r\n" or "--\r\n"
+					OutputStream dummy = new DummyOutputStream();
+					DataHelper.copy(eofin, dummy);  
+					if (!eofin.wasFound())
+						Debug.debug(Debug.DEBUG, "EOF hit before first boundary " + boundary);
+					if (readBoundaryTrailer(in)) {
+						if (!eofin.wasFound())
+							Debug.debug(Debug.DEBUG, "EOF hit before first part body " + boundary);
+						tmpEnd = (int) eofin.getRead();
+						break;
 					}
-				}					
+					// From here on we do include the \r
+					match = DataHelper.getASCII("\r\n--" + boundary);
+					eofin = new EOFOnMatchInputStream(in, counter, match);
+				}
+				MailPart newPart = new MailPart(uidl, buffer, eofin, eofin, null);
+				parts.add( newPart );
+				tmpEnd = (int) eofin.getRead();
+				if (!eofin.wasFound()) {
+					// if MailPart contains a MailPart, we may not have drained to the end
+					DataHelper.copy(eofin, DUMMY_OUTPUT);  
+					if (!eofin.wasFound())
+						Debug.debug(Debug.DEBUG, "EOF hit before end of body " + i + " boundary: " + boundary);
+				}
+				if (readBoundaryTrailer(in))
+					break;
 			}
 		}
 		else if( message ) {
-			MailPart newPart = new MailPart(uidl, buffer, beginBody, end - beginBody);
+			MailPart newPart = new MailPart(uidl, buffer, in, counter, null);
+			// TODO newPart doesn't save message headers we might like to display,
+			// like From, To, and Subject
 			parts.add( newPart );			
+			tmpEnd = (int) counter.getRead();
+		} else {
+			// read through to the end
+			DataHelper.copy(in, DUMMY_OUTPUT);  
+			tmpEnd = (int) counter.getRead();
+		}
+		end = tmpEnd;
+		if (encoding == null || encoding.equals("7bit") || encoding.equals("8bit")) {
+			decodedLength = end - beginBody;
 		}
+		//if (Debug.getLevel() >= Debug.DEBUG)
+		//	Debug.debug(Debug.DEBUG, "New " + this);
+	}
+
+	/**
+	 *  Swallow "\r\n" or "--\r\n".
+	 *  We don't have any pushback if this goes wrong.
+	 *
+	 *  @return true if end of input
+	 */
+	private static boolean readBoundaryTrailer(InputStream in) throws IOException {
+		int c = in.read();
+		if (c == '-') {
+			// end of parts with this boundary
+			c = in.read();
+			if (c != '-') {
+				Debug.debug(Debug.DEBUG, "Unexpected char after boundary-: " + c);
+				return true;
+			}
+			c = in.read();
+			if (c == -1) {
+				return true;
+			}
+			if (c != '\r') {
+				Debug.debug(Debug.DEBUG, "Unexpected char after boundary--: " + c);
+				return true;
+			}
+			c = in.read();
+			if (c != '\n')
+				Debug.debug(Debug.DEBUG, "Unexpected char after boundary--\\r: " + c);
+			return true;
+		} else if (c == '\r') {
+			c = in.read();
+			if (c != '\n')
+				Debug.debug(Debug.DEBUG, "Unexpected char after boundary\\r: " + c);
+		} else {
+			Debug.debug(Debug.DEBUG, "Unexpected char after boundary: " + c);
+		}
+		return c == -1;
 	}
 
 	/**
          *  @param offset 2 for sendAttachment, 0 otherwise, probably for \r\n
 	 *  @since 0.9.13
 	 */
-	public ReadBuffer decode(int offset) throws DecodingException {
+	public void decode(int offset, Buffer out) throws IOException {
 		String encg = encoding;
 		if (encg == null) {
 			//throw new DecodingException("No encoding specified");
@@ -213,7 +296,44 @@ class MailPart {
 		Encoding enc = EncodingFactory.getEncoding(encg);
 		if(enc == null)
 			throw new DecodingException(_t("No encoder found for encoding \\''{0}\\''.", WebMail.quoteHTML(encg)));
-		return enc.decode(buffer.content, beginBody + offset, end - beginBody - offset);
+		InputStream in = null;
+		LimitInputStream lin = null;
+		CountingOutputStream cos = null;
+		Buffer dout = null;
+		try {
+			in = buffer.getInputStream();
+			DataHelper.skip(in, buffer.getOffset() + beginBody + offset);
+			lin = new LimitInputStream(in, end - beginBody - offset);
+			if (decodedLength < 0) {
+				cos = new CountingOutputStream(out.getOutputStream());
+				dout = new OutputStreamBuffer(cos);
+			} else {
+	      			dout = out;
+			}
+			enc.decode(lin, dout);
+			//dout.getOutputStream().flush();
+		} catch (IOException ioe) {
+			if (lin != null)
+				Debug.debug(Debug.DEBUG, "Decode IOE at in position " + lin.getRead()
+				            + " offset " + offset, ioe);
+			else if (cos != null)
+				Debug.debug(Debug.DEBUG, "Decode IOE at out position " + cos.getWritten()
+				            + " offset " + offset, ioe);
+			else
+				Debug.debug(Debug.DEBUG, "Decode IOE", ioe);
+			throw ioe;
+		} finally {
+			if (in != null) try { in.close(); } catch (IOException ioe) {};
+			if (lin != null) try { lin.close(); } catch (IOException ioe) {};
+			buffer.readComplete(true);
+			// let the servlet do this
+			//if (cos != null) try { cos.close(); } catch (IOException ioe) {};
+			//if (dout != null)
+			//	dout.writeComplete(true);
+			//out.writeComplete(true);
+		}
+		if (cos != null)
+			decodedLength = (int) cos.getWritten();
 	}
 
 	private static String getFirstAttribute( String line )
@@ -305,4 +425,40 @@ class MailPart {
 	private static String _t(String s, Object o) {
 		return Messages.getString(s, o);
 	}
+
+	@Override
+	public String toString() {
+		StringBuilder buf = new StringBuilder(1024);
+		buf.append(
+			"MailPart:" +
+			"\n\tuidl:\t" + uidl +
+			"\n\tbuffer:\t" + buffer +
+			"\n\tbuffer offset:\t" + buffer.getOffset() +
+			"\n\tbegin:\t" + begin +
+			"\n\theader lines:\t" + headerLines.length +
+			"\n"
+		);
+		for (int i = 0; i < headerLines.length; i++) {
+			buf.append("\t\t\"").append(headerLines[i]).append("\"\n");
+		}
+		buf.append(
+			"\tmultipart?\t" + multipart +
+			"\n\tmessage?\t" + message +
+			"\n\ttype:\t" + type +
+			"\n\tencoding:\t" + encoding +
+			"\n\tname:\t" + name +
+			"\n\tdescription:\t" + description +
+			"\n\tdisposition:\t" + disposition +
+			"\n\tcharset:\t" + charset +
+			"\n\tversion:\t" + version +
+			"\n\tsubparts:\t" + parts.size() +
+			"\n\tbeginbody:\t" + beginBody +
+			"\n\tbody len:\t" + (end - beginBody) +
+			"\n\tdecoded len:\t" + decodedLength +
+			"\n\tend:\t" + (end - 1) +
+			"\n\ttotal len:\t" + (end - begin) +
+			"\n\tbuffer len:\t" + buffer.getLength()
+		);
+		return  buf.toString();
+	}
 }

apps/susimail/src/src/i2p/susi/webmail/PersistentMailCache.java

@@ -2,6 +2,9 @@ package i2p.susi.webmail;
 
 import i2p.susi.debug.Debug;
 import i2p.susi.webmail.Messages;
+import i2p.susi.util.Buffer;
+import i2p.susi.util.FileBuffer;
+import i2p.susi.util.GzipFileBuffer;
 import i2p.susi.util.ReadBuffer;
 
 import java.io.BufferedInputStream;
@@ -127,7 +130,7 @@ class PersistentMailCache {
 	private boolean locked_getMail(Mail mail, boolean headerOnly) {
 		File f = getFullFile(mail.uidl);
 		if (f.exists()) {
-			ReadBuffer rb = read(f);
+			Buffer rb = read(f);
 			if (rb != null) {
 				mail.setBody(rb);
 				return true;
@@ -135,7 +138,7 @@ class PersistentMailCache {
 		}
 		f = getHeaderFile(mail.uidl);
 		if (f.exists()) {
-			ReadBuffer rb = read(f);
+			Buffer rb = read(f);
 			if (rb != null) {
 				mail.setHeader(rb);
 				return true;
@@ -156,7 +159,7 @@ class PersistentMailCache {
 	}
 
 	private boolean locked_saveMail(Mail mail) {
-		ReadBuffer rb = mail.getBody();
+		Buffer rb = mail.getBody();
 		if (rb != null) {
 			File f = getFullFile(mail.uidl);
 			if (f.exists())
@@ -249,16 +252,21 @@ class PersistentMailCache {
 	 * 
 	 * @return success
 	 */
-	private static boolean write(ReadBuffer rb, File f) {
+	private static boolean write(Buffer rb, File f) {
+		InputStream in = null;
 		OutputStream out = null;
 		try {
-			out = new BufferedOutputStream(new GZIPOutputStream(new SecureFileOutputStream(f)));
-			out.write(rb.content, rb.offset, rb.length);
+			in = rb.getInputStream();
+			GzipFileBuffer gb = new GzipFileBuffer(f);
+			out = gb.getOutputStream();
+			DataHelper.copy(in, out);
 			return true;
 		} catch (IOException ioe) {
 			Debug.debug(Debug.ERROR, "Error writing: " + f + ": " + ioe);
 			return false;
 		} finally {
+			if (in != null) 
+				try { in.close(); } catch (IOException ioe) {}
 			if (out != null) 
 				try { out.close(); } catch (IOException ioe) {}
 		}
@@ -267,28 +275,8 @@ class PersistentMailCache {
 	/**
 	 *  @return null on failure
 	 */
-	private static ReadBuffer read(File f) {
-		InputStream in = null;
-		try {
-			long len = f.length();
-			if (len > 16 * 1024 * 1024) {
-				throw new IOException("too big");
-			}
-			in = new GZIPInputStream(new BufferedInputStream(new FileInputStream(f)));
-			ByteArrayOutputStream out = new ByteArrayOutputStream((int) len);
-			DataHelper.copy(in, out);
-			ReadBuffer rb = new ReadBuffer(out.toByteArray(), 0, out.size());
-			return rb;
-		} catch (IOException ioe) {
-			Debug.debug(Debug.ERROR, "Error reading: " + f + ": " + ioe);
-			return null;
-		} catch (OutOfMemoryError oom) {
-			Debug.debug(Debug.ERROR, "Error reading: " + f + ": " + oom);
-			return null;
-		} finally {
-			if (in != null) 
-				try { in.close(); } catch (IOException ioe) {}
-		}
+	private static Buffer read(File f) {
+		return new GzipFileBuffer(f);
 	}
 
 	/**
@@ -309,7 +297,7 @@ class PersistentMailCache {
 		}
 		if (uidl == null)
 			return null;
-		ReadBuffer rb = read(f);
+		Buffer rb = read(f);
 		if (rb == null)
 			return null;
 		Mail mail = new Mail(uidl);