From 8161f099d20ae5b5842648e51eeb36a1e9536df7 Mon Sep 17 00:00:00 2001 From: zzz Date: Thu, 8 Feb 2018 14:46:41 +0000 Subject: [PATCH] SusiMail: Error handling fixes More tolerant parsing of Date headers Set a date if we don't get a valid Date header Fix parsing long Base64 encoded headers Fix page count after changing page size Make attribute name parsing case-insensitive Import mail method for debugging Debug and log tweaks --- .../i2p/susi/util/FixCRLFOutputStream.java | 28 +++++++ .../src/src/i2p/susi/webmail/Mail.java | 45 ++++++---- .../src/src/i2p/susi/webmail/MailCache.java | 2 +- .../src/src/i2p/susi/webmail/MailPart.java | 13 ++- .../i2p/susi/webmail/PersistentMailCache.java | 83 ++++++++++++++++++- .../src/src/i2p/susi/webmail/WebMail.java | 37 +++++++-- .../i2p/susi/webmail/encoding/HeaderLine.java | 39 ++++++--- history.txt | 10 +++ .../src/net/i2p/router/RouterVersion.java | 2 +- 9 files changed, 216 insertions(+), 43 deletions(-) create mode 100644 apps/susimail/src/src/i2p/susi/util/FixCRLFOutputStream.java diff --git a/apps/susimail/src/src/i2p/susi/util/FixCRLFOutputStream.java b/apps/susimail/src/src/i2p/susi/util/FixCRLFOutputStream.java new file mode 100644 index 000000000..97f5a0439 --- /dev/null +++ b/apps/susimail/src/src/i2p/susi/util/FixCRLFOutputStream.java @@ -0,0 +1,28 @@ +package i2p.susi.util; + +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +/** + * Replace plain \n with \r\n on the fly. + * Used when importing .eml files. + * + * @since 0.9.34 + */ +public class FixCRLFOutputStream extends FilterOutputStream { + + private int previous = -1; + + public FixCRLFOutputStream(OutputStream out) { + super(out); + } + + @Override + public void write(int val) throws IOException { + if (val == '\n' && previous != '\r') + out.write('\r'); + out.write(val); + previous = val; + } +} diff --git a/apps/susimail/src/src/i2p/susi/webmail/Mail.java b/apps/susimail/src/src/i2p/susi/webmail/Mail.java index d1007bd2a..a1348ea62 100644 --- a/apps/susimail/src/src/i2p/susi/webmail/Mail.java +++ b/apps/susimail/src/src/i2p/susi/webmail/Mail.java @@ -28,6 +28,7 @@ import i2p.susi.util.Buffer; import i2p.susi.util.Config; import i2p.susi.util.CountingInputStream; import i2p.susi.util.EOFOnMatchInputStream; +import i2p.susi.util.FileBuffer; import i2p.susi.util.MemoryBuffer; import i2p.susi.webmail.encoding.Encoding; import i2p.susi.webmail.encoding.EncodingFactory; @@ -46,8 +47,10 @@ import java.util.Locale; import java.util.TimeZone; import java.util.regex.Pattern; +import net.i2p.I2PAppContext; import net.i2p.data.DataHelper; import net.i2p.servlet.util.ServletUtil; +import net.i2p.util.RFC822Date; import net.i2p.util.SystemVersion; /** @@ -129,6 +132,16 @@ class Mail { String[] rv = parseHeaders(in); if (closeIn) rb.readComplete(true); + // set a date if we didn't get one in the headers + if (date == null) { + long dateLong; + if (rb instanceof FileBuffer) { + dateLong = ((FileBuffer) rb).getFile().lastModified(); + } else { + dateLong = I2PAppContext.getGlobalContext().clock().now(); + } + setDate(dateLong); + } return rv; } @@ -175,7 +188,7 @@ class Mail { } catch (RuntimeException e) { Debug.debug(Debug.ERROR, "Parse error", e); } finally { - try { in.close(); } catch (IOException ioe) {} + if (in != null) try { in.close(); } catch (IOException ioe) {} rb.readComplete(success); } } @@ -323,7 +336,6 @@ class Mail { private static final DateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd HH:mm"); private static final DateFormat localDateFormatter = DateFormat.getDateTimeInstance(DateFormat.SHORT, DateFormat.SHORT); private static final DateFormat longLocalDateFormatter = DateFormat.getDateTimeInstance(DateFormat.MEDIUM, DateFormat.MEDIUM); - private static final DateFormat mailDateFormatter = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss Z", Locale.ENGLISH ); static { // the router sets the JVM time zone to UTC but saves the original here so we can get it TimeZone tz = SystemVersion.getSystemTimeZone(); @@ -331,6 +343,19 @@ class Mail { longLocalDateFormatter.setTimeZone(tz); } + /** + * @param dateLong non-negative + * @since 0.9.34 pulled from parseHeaders() + */ + private void setDate(long dateLong) { + date = new Date(dateLong); + synchronized(dateFormatter) { + formattedDate = dateFormatter.format( date ); + localFormattedDate = localDateFormatter.format( date ); + quotedDate = longLocalDateFormatter.format(date); + } + } + /** * @return all headers, to pass to MailPart, or null on error */ @@ -390,19 +415,9 @@ class Mail { } else if (hlc.startsWith("date:")) { dateString = line.substring( 5 ).trim(); - try { - synchronized(mailDateFormatter) { - date = mailDateFormatter.parse( dateString ); - formattedDate = dateFormatter.format( date ); - localFormattedDate = localDateFormatter.format( date ); - //quotedDate = html.encode( dateString ); - quotedDate = longLocalDateFormatter.format(date); - } - } - catch (ParseException e) { - date = null; - e.printStackTrace(); - } + long dateLong = RFC822Date.parse822Date(dateString); + if (dateLong > 0) + setDate(dateLong); } else if (hlc.startsWith("subject:")) { subject = line.substring( 8 ).trim(); diff --git a/apps/susimail/src/src/i2p/susi/webmail/MailCache.java b/apps/susimail/src/src/i2p/susi/webmail/MailCache.java index 3394f8aa7..053317dbf 100644 --- a/apps/susimail/src/src/i2p/susi/webmail/MailCache.java +++ b/apps/susimail/src/src/i2p/susi/webmail/MailCache.java @@ -72,7 +72,7 @@ class MailCache { mails = new Hashtable(); PersistentMailCache pmc = null; try { - pmc = new PersistentMailCache(host, port, user, pass, PersistentMailCache.DIR_FOLDER); + pmc = new PersistentMailCache(ctx, host, port, user, pass, PersistentMailCache.DIR_FOLDER); // TODO Drafts, Sent, Trash } catch (IOException ioe) { Debug.debug(Debug.ERROR, "Error creating disk cache: " + ioe); diff --git a/apps/susimail/src/src/i2p/susi/webmail/MailPart.java b/apps/susimail/src/src/i2p/susi/webmail/MailPart.java index 49cda3d94..2068cd198 100644 --- a/apps/susimail/src/src/i2p/susi/webmail/MailPart.java +++ b/apps/susimail/src/src/i2p/susi/webmail/MailPart.java @@ -202,10 +202,10 @@ class MailPart { OutputStream dummy = new DummyOutputStream(); DataHelper.copy(eofin, dummy); if (!eofin.wasFound()) - Debug.debug(Debug.DEBUG, "EOF hit before first boundary " + boundary); + Debug.debug(Debug.DEBUG, "EOF hit before first boundary " + boundary + " UIDL: " + uidl); if (readBoundaryTrailer(in)) { if (!eofin.wasFound()) - Debug.debug(Debug.DEBUG, "EOF hit before first part body " + boundary); + Debug.debug(Debug.DEBUG, "EOF hit before first part body " + boundary + " UIDL: " + uidl); tmpEnd = (int) eofin.getRead(); break; } @@ -220,7 +220,7 @@ class MailPart { // if MailPart contains a MailPart, we may not have drained to the end DataHelper.copy(eofin, DUMMY_OUTPUT); if (!eofin.wasFound()) - Debug.debug(Debug.DEBUG, "EOF hit before end of body " + i + " boundary: " + boundary); + Debug.debug(Debug.DEBUG, "EOF hit before end of body " + i + " boundary: " + boundary + " UIDL: " + uidl); } if (readBoundaryTrailer(in)) break; @@ -351,13 +351,18 @@ class MailPart { return result; } + /** + * @param attributeName must be lower case, will be matched case-insensitively + * @return as found, not necessarily lower case + */ private static String getHeaderLineAttribute( String line, String attributeName ) { + String lineLC = line.toLowerCase(Locale.US); String result = null; int h = 0; int l = attributeName.length(); while( true ) { - int i = line.indexOf( attributeName, h ); + int i = lineLC.indexOf(attributeName, h); // System.err.println( "i=" + i ); if( i == -1 ) break; diff --git a/apps/susimail/src/src/i2p/susi/webmail/PersistentMailCache.java b/apps/susimail/src/src/i2p/susi/webmail/PersistentMailCache.java index 653927264..7df5acccc 100644 --- a/apps/susimail/src/src/i2p/susi/webmail/PersistentMailCache.java +++ b/apps/susimail/src/src/i2p/susi/webmail/PersistentMailCache.java @@ -4,6 +4,7 @@ import i2p.susi.debug.Debug; import i2p.susi.webmail.Messages; import i2p.susi.util.Buffer; import i2p.susi.util.FileBuffer; +import i2p.susi.util.FixCRLFOutputStream; import i2p.susi.util.GzipFileBuffer; import i2p.susi.util.ReadBuffer; @@ -19,6 +20,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Hashtable; import java.util.List; +import java.util.Locale; import java.util.concurrent.ConcurrentHashMap; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; @@ -60,6 +62,7 @@ class PersistentMailCache { private final Object _lock; private final File _cacheDir; + private final I2PAppContext _context; private static final String DIR_SUSI = "susimail"; private static final String DIR_CACHE = "cache"; @@ -68,6 +71,8 @@ class PersistentMailCache { public static final String DIR_DRAFTS = "Drafts"; // MailDir-like public static final String DIR_SENT = "Sent"; // MailDir-like public static final String DIR_TRASH = "Trash"; // MailDir-like + public static final String DIR_SPAM = "Bulk Mail"; // MailDir-like + public static final String DIR_IMPORT = "import"; // Flat with .eml files, debug only for now private static final String DIR_PREFIX = "s"; private static final String FILE_PREFIX = "mail-"; private static final String HDR_SUFFIX = ".hdr.txt.gz"; @@ -79,10 +84,14 @@ class PersistentMailCache { * @param pass ignored * @param folder use DIR_FOLDER */ - public PersistentMailCache(String host, int port, String user, String pass, String folder) throws IOException { + public PersistentMailCache(I2PAppContext ctx, String host, int port, String user, String pass, String folder) throws IOException { + _context = ctx; _lock = getLock(host, port, user, pass); synchronized(_lock) { _cacheDir = makeCacheDirs(host, port, user, pass, folder); + // Debugging only for now. + if (folder.equals(DIR_FOLDER)) + importMail(); } } @@ -209,8 +218,8 @@ class PersistentMailCache { * ~/.i2p/susimail/cache/cache-xxxxx/cur/s[b64char]/mail-xxxxx.full.txt.gz * folder1 is the base. */ - private static File makeCacheDirs(String host, int port, String user, String pass, String folder) throws IOException { - File f = new SecureDirectory(I2PAppContext.getGlobalContext().getConfigDir(), DIR_SUSI); + private File makeCacheDirs(String host, int port, String user, String pass, String folder) throws IOException { + File f = new SecureDirectory(_context.getConfigDir(), DIR_SUSI); if (!f.exists() && !f.mkdir()) throw new IOException("Cannot create " + f); f = new SecureDirectory(f, DIR_CACHE); @@ -307,4 +316,72 @@ class PersistentMailCache { mail.setBody(rb); return mail; } + + /** + * For debugging. Import .eml files from the import/ directory + * @since 0.9.34 + */ + private void importMail() { + File importDir = new File(_cacheDir.getParentFile(), DIR_IMPORT); + if (importDir.exists() && importDir.isDirectory()) { + File[] files = importDir.listFiles(); + if (files == null) + return; + for (int i = 0; i < files.length; i++) { + File f = files[i]; + if (!f.isFile()) + continue; + if (!f.getName().toLowerCase(Locale.US).endsWith(".eml")) + continue; + // Read in the headers to get the X-UIDL that Thunderbird stuck in there + String uidl = Long.toString(_context.random().nextLong()); + InputStream in = null; + try { + in = new FileInputStream(f); + for (int j = 0; j < 20; j++) { + String line = DataHelper.readLine(in); + if (line.length() < 2) + break; + if (line.startsWith("X-UIDL:")) { + uidl = line.substring(7).trim(); + break; + } + } + } catch (IOException ioe) { + Debug.debug(Debug.ERROR, "Import failed " + f, ioe); + continue; + } finally { + if (in != null) + try { in.close(); } catch (IOException ioe) {} + } + if (uidl == null) + uidl = Long.toString(_context.random().nextLong()); + File to = getFullFile(uidl); + if (to.exists()) { + Debug.debug(Debug.DEBUG, "Already have " + f + " as UIDL " + uidl); + f.delete(); + continue; + } + in = null; + OutputStream out = null; + try { + in = new FileInputStream(f); + GzipFileBuffer gb = new GzipFileBuffer(to); + // Thunderbird exports aren't CRLF terminated + out = new FixCRLFOutputStream(gb.getOutputStream()); + DataHelper.copy(in, out); + } catch (IOException ioe) { + Debug.debug(Debug.ERROR, "Import failed " + f, ioe); + continue; + } finally { + if (in != null) + try { in.close(); } catch (IOException ioe) {} + if (out != null) + try { out.close(); } catch (IOException ioe) {} + } + f.delete(); + Debug.debug(Debug.DEBUG, "Imported " + f + " as UIDL " + uidl); + } + } + } } diff --git a/apps/susimail/src/src/i2p/susi/webmail/WebMail.java b/apps/susimail/src/src/i2p/susi/webmail/WebMail.java index 87f9f6fe9..4f22898ac 100644 --- a/apps/susimail/src/src/i2p/susi/webmail/WebMail.java +++ b/apps/susimail/src/src/i2p/susi/webmail/WebMail.java @@ -635,6 +635,20 @@ public class WebMail extends HttpServlet } if( chosen != null ) { showPart( out, chosen, level + 1, html ); + if (html) { + // DEBUG + for (MailPart subPart : mailPart.parts) { + if (chosen.equals(subPart)) + continue; + out.println( "" ); + } + } return; } } @@ -1499,6 +1513,7 @@ public class WebMail extends HttpServlet /* * process paging buttons */ +/**** not on the folder view any more, handled in processConfigButtons() if (buttonPressed(request, SETPAGESIZE)) { try { int pageSize = Math.max(5, Integer.parseInt(request.getParameter(PAGESIZE))); @@ -1510,6 +1525,7 @@ public class WebMail extends HttpServlet sessionObject.error += _t("Invalid pagesize number, resetting to default value.") + '\n'; } } +****/ if( buttonPressed( request, PREVPAGE ) ) { String sp = request.getParameter(PREV_PAGE_NUM); if (sp != null) { @@ -1626,7 +1642,6 @@ public class WebMail extends HttpServlet File cfg = new File(I2PAppContext.getGlobalContext().getConfigDir(), "susimail.config"); sessionObject.error += _t("Host unchanged. Edit configation file {0} to change host.", cfg.getAbsolutePath()) + '\n'; } - Config.saveConfiguration(props); String ps = props.getProperty(Folder.PAGESIZE); if (sessionObject.folder != null && ps != null) { try { @@ -1636,6 +1651,7 @@ public class WebMail extends HttpServlet sessionObject.folder.setPageSize( pageSize ); } catch( NumberFormatException nfe ) {} } + Config.saveConfiguration(props); boolean release = !Boolean.parseBoolean(props.getProperty(CONFIG_DEBUG)); Debug.setLevel( release ? Debug.ERROR : Debug.DEBUG ); state = sessionObject.folder != null ? State.LIST : State.AUTH; @@ -1646,9 +1662,6 @@ public class WebMail extends HttpServlet } else if (buttonPressed(request, SETPAGESIZE)) { try { int pageSize = Math.max(5, Integer.parseInt(request.getParameter(PAGESIZE))); - Properties props = Config.getProperties(); - props.setProperty(Folder.PAGESIZE, String.valueOf(pageSize)); - Config.saveConfiguration(props); if (sessionObject.folder != null) { int oldPageSize = sessionObject.folder.getPageSize(); if( pageSize != oldPageSize ) @@ -1657,6 +1670,9 @@ public class WebMail extends HttpServlet } else { state = State.AUTH; } + Properties props = Config.getProperties(); + props.setProperty(Folder.PAGESIZE, String.valueOf(pageSize)); + Config.saveConfiguration(props); } catch (IOException ioe) { sessionObject.error = ioe.toString(); } catch( NumberFormatException nfe ) { @@ -1897,15 +1913,22 @@ public class WebMail extends HttpServlet } } - //// End state determination, state will not change after here - Debug.debug(Debug.DEBUG, "Final state is " + state); - /* * update folder content * We need a valid and sorted folder for SHOW also, for the previous/next buttons */ Folder folder = sessionObject.folder; + // folder could be null after an error, we can't proceed if it is + if (folder == null && (state == State.LIST || state == State.SHOW)) { + sessionObject.error += "Internal error, no folder\n"; + state = State.AUTH; + } + + //// End state determination, state will not change after here + Debug.debug(Debug.DEBUG, "Final state is " + state); + if (state == State.LIST || state == State.SHOW) { + // sort buttons are GETs String oldSort = folder.getCurrentSortBy(); SortOrder oldOrder = folder.getCurrentSortingDirection(); diff --git a/apps/susimail/src/src/i2p/susi/webmail/encoding/HeaderLine.java b/apps/susimail/src/src/i2p/susi/webmail/encoding/HeaderLine.java index d7a1e0fbe..c268fea36 100644 --- a/apps/susimail/src/src/i2p/susi/webmail/encoding/HeaderLine.java +++ b/apps/susimail/src/src/i2p/susi/webmail/encoding/HeaderLine.java @@ -29,6 +29,7 @@ import i2p.susi.util.Buffer; import i2p.susi.util.ReadBuffer; import i2p.susi.util.MemoryBuffer; +import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -210,6 +211,9 @@ public class HeaderLine extends Encoding { return out.toString(); } + // could be 75 for quoted-printable only + private static final int DECODE_MAX = 256; + /** * Decode all the header lines, up through \r\n\r\n, * and puts them in the ReadBuffer, including the \r\n\r\n @@ -235,10 +239,11 @@ public class HeaderLine extends Encoding { break; } if( c == '=' ) { - // An encoded-word is 75 chars max including the delimiters, and must be on a single line + // An encoded-word should be 75 chars max including the delimiters, and must be on a single line // Store the full encoded word, including =? through ?=, in the buffer + // Sadly, base64 can be a lot longer if (encodedWord == null) - encodedWord = new byte[75]; + encodedWord = new byte[DECODE_MAX]; int offset = 0; int f1 = 0, f2 = 0, f3 = 0, f4 = 0; encodedWord[offset++] = (byte) c; @@ -246,7 +251,7 @@ public class HeaderLine extends Encoding { // plus one char after the 4th '?', which should be '=' // We make a small attempt to pushback one char if it's not what we expect, // but for the most part it gets thrown out, as RFC 2047 allows - for (; offset < 75; offset++) { + for (; offset < DECODE_MAX; offset++) { c = in.read(); if (c == '?') { if (f1 == 0) @@ -318,12 +323,20 @@ public class HeaderLine extends Encoding { if (enc != null) { Encoding e = EncodingFactory.getEncoding( enc ); if( e != null ) { - // System.err.println( "encoder found" ); try { // System.err.println( "decode(" + (f3 + 1) + "," + ( f4 - f3 - 1 ) + ")" ); ReadBuffer tmpIn = new ReadBuffer(encodedWord, f3 + 1, f4 - f3 - 1); - MemoryBuffer tmp = new MemoryBuffer(75); - e.decode(tmpIn, tmp); + MemoryBuffer tmp = new MemoryBuffer(DECODE_MAX); + try { + e.decode(tmpIn, tmp); + } catch (EOFException eof) { + // probably Base64 exceeded DECODE_MAX + // Keep going and output what we got, if any + if (Debug.getLevel() >= Debug.DEBUG) { + Debug.debug(Debug.DEBUG, "q-w " + enc, eof); + Debug.debug(Debug.DEBUG, net.i2p.util.HexDump.dump(encodedWord)); + } + } tmp.writeComplete(true); // get charset String charset = new String(encodedWord, f1 + 1, f2 - f1 - 1, "ISO-8859-1"); @@ -357,13 +370,15 @@ public class HeaderLine extends Encoding { lastCharWasQuoted = true; continue; } catch (IOException e1) { - Debug.debug(Debug.DEBUG, "q-w", e1); - Debug.debug(Debug.DEBUG, "Decoder: " + enc + " Input:"); - Debug.debug(Debug.DEBUG, net.i2p.util.HexDump.dump(encodedWord, f3 + 1, f4 - f3 - 1)); + Debug.debug(Debug.ERROR, "q-w " + enc, e1); + if (Debug.getLevel() >= Debug.DEBUG) { + Debug.debug(Debug.DEBUG, net.i2p.util.HexDump.dump(encodedWord)); + } } catch (RuntimeException e1) { - Debug.debug(Debug.DEBUG, "q-w", e1); - Debug.debug(Debug.DEBUG, "Decoder: " + enc + " Input:"); - Debug.debug(Debug.DEBUG, net.i2p.util.HexDump.dump(encodedWord, f3 + 1, f4 - f3 - 1)); + Debug.debug(Debug.ERROR, "q-w " + enc, e1); + if (Debug.getLevel() >= Debug.DEBUG) { + Debug.debug(Debug.DEBUG, net.i2p.util.HexDump.dump(encodedWord)); + } } } else { // can't happen diff --git a/history.txt b/history.txt index 01d9e1bdc..09f78fdb1 100644 --- a/history.txt +++ b/history.txt @@ -1,3 +1,13 @@ +2018-02-08 zzz + * SusiMail: + - Error handling fixes + - More tolerant parsing of Date headers + - Set a date if we don't get a Date header + - Fix parsing long Base64 encoded headers + - Fix page count after changing page size + - Make attribute name parsing case-insensitive + - Import mail method for debugging + 2018-02-07 zzz * SusiMail: Use input streams for reading mail (ticket #2119) - Rewrite Base64, HeaderLine, and QuotedPrintable decoders diff --git a/router/java/src/net/i2p/router/RouterVersion.java b/router/java/src/net/i2p/router/RouterVersion.java index 282c18b42..0725033fa 100644 --- a/router/java/src/net/i2p/router/RouterVersion.java +++ b/router/java/src/net/i2p/router/RouterVersion.java @@ -18,7 +18,7 @@ public class RouterVersion { /** deprecated */ public final static String ID = "Monotone"; public final static String VERSION = CoreVersion.VERSION; - public final static long BUILD = 2; + public final static long BUILD = 3; /** for example "-test" */ public final static String EXTRA = "";