* implemented fragmentation

* added more inbound tests
* made the tunnel preprocessing header more clear and included better fragmentation support
(still left: tests for outbound tunnel processing, structures and jobs to integrate with the router,
remove that full SHA256 from each and every I2NPMessage or put a smaller one at the
transport layer, and all the rest of the tunnel pooling/building stuff)

router/doc/tunnel-alt.html

-<code>$Id: tunnel-alt.html,v 1.4 2005/01/19 01:24:25 jrandom Exp $</code>
+<code>$Id: tunnel-alt.html,v 1.5 2005/01/19 18:13:10 jrandom Exp $</code>
 <pre>
 1) <a href="#tunnel.overview">Tunnel overview</a>
 2) <a href="#tunnel.operation">Tunnel operation</a>
@@ -111,27 +111,42 @@ data into the raw tunnel payload:</p>
 <li>a series of zero or more { instructions, message } pairs</li>
 </ul>
 
-<p>The instructions are encoded as follows:</p>
+<p>The instructions are encoded with a single control byte, followed by any
+necessary additional information.  The first bit in that control byte determines
+how the remainder of the header is interpreted - if it is not set, the message 
+is eithernot fragmented or this is the first fragment in the message.  If it is
+set, this is a follow on fragment.</p>
+
+<p>With the first bit being 0, the instructions are:</p>
 <ul>
-<li>1 byte value:<pre>
-   bits 0-1: delivery type
+<li>1 byte control byte:<pre>
+      bit 0: is follow on fragment?  (1 = true, 0 = false, must be 0)
+   bits 1-2: delivery type
              (0x0 = LOCAL, 0x01 = TUNNEL, 0x02 = ROUTER)
-      bit 2: delay included?  (1 = true, 0 = false)
-      bit 3: fragmented?  (1 = true, 0 = false)
-      bit 4: extended options?  (1 = true, 0 = false)
-   bits 5-7: reserved</pre></li>
+      bit 3: delay included?  (1 = true, 0 = false)
+      bit 4: fragmented?  (1 = true, 0 = false)
+      bit 5: extended options?  (1 = true, 0 = false)
+   bits 6-7: reserved</pre></li>
 <li>if the delivery type was TUNNEL, a 4 byte tunnel ID</li>
 <li>if the delivery type was TUNNEL or ROUTER, a 32 byte router hash</li>
 <li>if the delay included flag is true, a 1 byte value:<pre>
       bit 0: type (0 = strict, 1 = randomized)
    bits 1-7: delay exponent (2^value minutes)</pre></li>
-<li>if the fragmented flag is true, a 4 byte message ID, and a 1 byte value:<pre>
-   bits 0-6: fragment number
-      bit 7: is last?  (1 = true, 0 = false)</pre></li>
+<li>if the fragmented flag is true, a 4 byte message ID</li>
 <li>if the extended options flag is true:<pre>
    = a 1 byte option size (in bytes)
    = that many bytes</pre></li>
-<li>2 byte size of the I2NP message</li>
+<li>2 byte size of the I2NP message or this fragment</li>
+</ul>
+
+<p>If the first bit being 1, the instructions are:</p>
+<ul>
+<li>1 byte control byte:<pre>
+      bit 0: is follow on fragment?  (1 = true, 0 = false, must be 1)
+   bits 1-6: fragment number
+      bit 7: is last? (1 = true, 0 = false)</pre></li>
+<li>4 byte message ID (same one defined in the first fragment)</li>
+<li>2 byte size of this fragment</li>
 </ul>
 
 <p>The I2NP message is encoded in its standard form, and the 
@@ -202,7 +217,8 @@ implement fragmentation.  Padding to the closest exponential size (ala freenet)
 seems promising.  Perhaps we should gather some stats on the net as to what size
 messages are, then see what costs and benefits would arise from different 
 strategies?  <b>See <a href="http://dev.i2p.net/~jrandom/messageSizes/">gathered
-stats</a></b></i></p>
+stats</a></b>.  The current plan is to pad to a fixed 1024 byte message size with
+fragmentation.</i></p>
 
 <h3>2.6) <a name="tunnel.fragmentation">Tunnel fragmentation</a></h3>
 

router/java/src/net/i2p/router/tunnel/FragmentHandler.java

+package net.i2p.router.tunnel;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import net.i2p.I2PAppContext;
+import net.i2p.data.Base64;
+import net.i2p.data.DataHelper;
+import net.i2p.data.Hash;
+import net.i2p.data.TunnelId;
+import net.i2p.data.i2np.I2NPMessage;
+import net.i2p.data.i2np.I2NPMessageException;
+import net.i2p.data.i2np.I2NPMessageHandler;
+import net.i2p.util.Log;
+import net.i2p.util.SimpleTimer;
+
+/**
+ * Handle fragments at the endpoint of a tunnel, peeling off fully completed 
+ * I2NPMessages when they arrive, and dropping fragments if they take too long
+ * to arrive.
+ *
+ */
+public class FragmentHandler {
+    private I2PAppContext _context;
+    private Log _log;
+    private Map _fragmentedMessages;
+    private DefragmentedReceiver _receiver;
+    
+    /** don't wait more than 20s to defragment the partial message */
+    private static final long MAX_DEFRAGMENT_TIME = 20*1000; 
+    
+    public FragmentHandler(I2PAppContext context, DefragmentedReceiver receiver) {
+        _context = context;
+        _log = context.logManager().getLog(FragmentHandler.class);
+        _fragmentedMessages = new HashMap(4);
+        _receiver = receiver;
+    }
+    
+    /**
+     * Receive the raw preprocessed message at the endpoint, parsing out each
+     * of the fragments, using those to fill various FragmentedMessages, and 
+     * sending the resulting I2NPMessages where necessary.  The received 
+     * fragments are all verified.
+     *
+     */
+    public void receiveTunnelMessage(byte preprocessed[], int offset, int length) {
+        boolean ok = verifyPreprocessed(preprocessed, offset, length);
+        if (!ok) {
+            _log.error("Unable to verify preprocessed data");
+            return;
+        }
+        offset += HopProcessor.IV_LENGTH; // skip the IV
+        offset += 4; // skip the hash segment
+        int padding = 0;
+        while (preprocessed[offset] != (byte)0x00) {
+            offset++; // skip the padding
+            padding++;
+        }
+        offset++; // skip the final 0x00, terminating the padding
+        if (_log.shouldLog(Log.DEBUG)) {
+            _log.debug("Fragments begin at offset=" + offset + " padding=" + padding);
+            _log.debug("fragments: " + Base64.encode(preprocessed, offset, preprocessed.length-offset));
+        }
+        try {
+            while (offset < length)
+                offset = receiveFragment(preprocessed, offset, length);
+        } catch (Exception e) {
+            if (_log.shouldLog(Log.ERROR))
+                _log.error("Corrupt fragment received: offset = " + offset, e);
+        }
+    }
+    
+    /**
+     * Verify that the preprocessed data hasn't been modified by checking the 
+     * H(payload+IV)[0:3] vs preprocessed[16:19], where payload is the data 
+     * after the padding.  Remember, the preprocessed data is formatted as
+     * { IV + H[0:3] + padding + {instructions, fragment}* }.  This function is
+     * very wasteful of memory usage as it doesn't operate inline (since IV and
+     * payload are mixed up).  Later it may be worthwhile to explore optimizing
+     * this.
+     */
+    private boolean verifyPreprocessed(byte preprocessed[], int offset, int length) {
+        // now we need to verify that the message was received correctly
+        int paddingEnd = HopProcessor.IV_LENGTH + 4;
+        while (preprocessed[offset+paddingEnd] != (byte)0x00)
+            paddingEnd++;
+        paddingEnd++; // skip the last
+        
+        byte preV[] = new byte[length - offset - paddingEnd + HopProcessor.IV_LENGTH];
+        System.arraycopy(preprocessed, offset + paddingEnd, preV, 0, preV.length - HopProcessor.IV_LENGTH);
+        System.arraycopy(preprocessed, 0, preV, preV.length - HopProcessor.IV_LENGTH, HopProcessor.IV_LENGTH);
+        Hash v = _context.sha().calculateHash(preV);
+        boolean eq = DataHelper.eq(v.getData(), 0, preprocessed, offset + HopProcessor.IV_LENGTH, 4);
+        if (!eq) 
+            _log.error("Endpoint data doesn't match:\n" + Base64.encode(preprocessed, offset + paddingEnd, preV.length-HopProcessor.IV_LENGTH));
+        return eq;
+    }
+    
+    /** is this a follw up byte? */
+    static final byte MASK_IS_SUBSEQUENT = (byte)(1 << 7);
+    /** how should this be delivered?  shift this 5 the right and get TYPE_* */
+    static final byte MASK_TYPE = (byte)(3 << 5);
+    /** is this the first of a fragmented message? */
+    static final byte MASK_FRAGMENTED = (byte)(1 << 3);
+    /** are there follow up headers? */
+    static final byte MASK_EXTENDED = (byte)(1 << 2);
+    /** for subsequent fragments, which bits contain the fragment #? */
+    private static final int MASK_FRAGMENT_NUM = (byte)((1 << 7) - 2); // 0x7E;
+    
+    static final short TYPE_LOCAL = 0;
+    static final short TYPE_TUNNEL = 1;
+    static final short TYPE_ROUTER = 2;
+    
+    /** 
+     * @return the offset for the next byte after the received fragment 
+     */
+    private int receiveFragment(byte preprocessed[], int offset, int length) {
+        if (_log.shouldLog(Log.DEBUG))
+            _log.debug("CONTROL: " + Integer.toHexString(preprocessed[offset]) + " / " 
+                       + "/" + Base64.encode(preprocessed, offset, 1) + " at offset " + offset);
+        if (0 == (preprocessed[offset] & MASK_IS_SUBSEQUENT))
+            return receiveInitialFragment(preprocessed, offset, length);
+        else
+            return receiveSubsequentFragment(preprocessed, offset, length);
+    }
+    
+    /**
+     * Handle the initial fragment in a message (or a full message, if it fits)
+     *
+     * @return offset after reading the full fragment
+     */
+    private int receiveInitialFragment(byte preprocessed[], int offset, int length) {
+        if (_log.shouldLog(Log.DEBUG))
+            _log.debug("initial begins at " + offset + " for " + length);
+        int type = (preprocessed[offset] & MASK_TYPE) >>> 5;
+        boolean fragmented = (0 != (preprocessed[offset] & MASK_FRAGMENTED));
+        boolean extended = (0 != (preprocessed[offset] & MASK_EXTENDED));
+        offset++;
+        
+        TunnelId tunnelId = null;
+        Hash router = null;
+        long messageId = -1;
+        
+        if (type == TYPE_TUNNEL) {
+            long id = DataHelper.fromLong(preprocessed, offset, 4);
+            tunnelId = new TunnelId(id);
+            offset += 4;
+        }
+        if ( (type == TYPE_ROUTER) || (type == TYPE_TUNNEL) ) {
+            byte h[] = new byte[Hash.HASH_LENGTH];
+            System.arraycopy(preprocessed, offset, h, 0, Hash.HASH_LENGTH);
+            router = new Hash(h);
+            offset += Hash.HASH_LENGTH;
+        }
+        if (fragmented) {
+            messageId = DataHelper.fromLong(preprocessed, offset, 4);
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("reading messageId " + messageId + " at offset "+ offset 
+                           + " type = " + type + "tunnelId = " + tunnelId);
+            offset += 4;
+        }
+        if (extended) {
+            int extendedSize = (int)DataHelper.fromLong(preprocessed, offset, 1);
+            offset++;
+            offset += extendedSize; // we don't interpret these yet, but skip them for now
+        }
+        
+        int size = (int)DataHelper.fromLong(preprocessed, offset, 2);
+        offset += 2;
+        
+        boolean isNew = false;
+        FragmentedMessage msg = null;
+        if (fragmented) {
+            synchronized (_fragmentedMessages) {
+                msg = (FragmentedMessage)_fragmentedMessages.get(new Long(messageId));
+                if (msg == null) {
+                    msg = new FragmentedMessage(_context);
+                    _fragmentedMessages.put(new Long(messageId), msg);
+                    isNew = true;
+                }
+            }
+        } else {
+            msg = new FragmentedMessage(_context);
+        }
+        
+        if (isNew && fragmented) {
+            RemoveFailed evt = new RemoveFailed(msg);
+            msg.setExpireEvent(evt);
+            _log.debug("In " + MAX_DEFRAGMENT_TIME + " dropping " + messageId);
+            SimpleTimer.getInstance().addEvent(evt, MAX_DEFRAGMENT_TIME);
+        }
+        
+        msg.receive(messageId, preprocessed, offset, size, !fragmented, router, tunnelId);
+        if (msg.isComplete()) {
+            if (fragmented) {
+                synchronized (_fragmentedMessages) {
+                    _fragmentedMessages.remove(new Long(messageId));
+                }
+            }
+            if (msg.getExpireEvent() != null)
+                SimpleTimer.getInstance().removeEvent(msg.getExpireEvent());
+            receiveComplete(msg);
+        }
+        
+        offset += size;
+        return offset;
+    }
+    
+    /**
+     * Handle a fragment beyond the initial fragment in a message
+     *
+     * @return offset after reading the full fragment
+     */
+    private int receiveSubsequentFragment(byte preprocessed[], int offset, int length) {
+        if (_log.shouldLog(Log.DEBUG))
+            _log.debug("subsequent begins at " + offset + " for " + length);
+        int fragmentNum = ((preprocessed[offset] & MASK_FRAGMENT_NUM) >>> 1);
+        boolean isLast = (0 != (preprocessed[offset] & 1));
+        offset++;
+        
+        long messageId = DataHelper.fromLong(preprocessed, offset, 4);
+        offset += 4;
+        
+        int size = (int)DataHelper.fromLong(preprocessed, offset, 2);
+        offset += 2;
+        
+        boolean isNew = false;
+        FragmentedMessage msg = null;
+        synchronized (_fragmentedMessages) {
+            msg = (FragmentedMessage)_fragmentedMessages.get(new Long(messageId));
+            if (msg == null) {
+                msg = new FragmentedMessage(_context);
+                _fragmentedMessages.put(new Long(messageId), msg);
+                isNew = true;
+            }
+        }
+        
+        if (isNew) {
+            RemoveFailed evt = new RemoveFailed(msg);
+            msg.setExpireEvent(evt);
+            _log.debug("In " + MAX_DEFRAGMENT_TIME + " dropping " + msg.getMessageId() + "/" + fragmentNum);
+            SimpleTimer.getInstance().addEvent(evt, MAX_DEFRAGMENT_TIME);
+        }
+        
+        msg.receive(messageId, fragmentNum, preprocessed, offset, size, isLast);
+        
+        if (msg.isComplete()) {
+            synchronized (_fragmentedMessages) {
+                _fragmentedMessages.remove(new Long(messageId));
+            }
+            if (msg.getExpireEvent() != null)
+                SimpleTimer.getInstance().removeEvent(msg.getExpireEvent());
+            receiveComplete(msg);
+        }
+        
+        offset += size;
+        return offset;
+    }
+    
+    private void receiveComplete(FragmentedMessage msg) {
+        try {
+            byte data[] = msg.toByteArray();
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("RECV(" + data.length + "): " + Base64.encode(data)  
+                           + " " + _context.sha().calculateHash(data).toBase64());
+            I2NPMessage m = new I2NPMessageHandler(_context).readMessage(data);
+            _receiver.receiveComplete(m, msg.getTargetRouter(), msg.getTargetTunnel());
+        } catch (IOException ioe) {
+            if (_log.shouldLog(Log.WARN))
+                _log.warn("Error receiving fragmented message (corrupt?): " + msg, ioe);
+        } catch (I2NPMessageException ime) {
+            if (_log.shouldLog(Log.WARN))
+                _log.warn("Error receiving fragmented message (corrupt?): " + msg, ime);
+        }
+    }
+
+    /**
+     * Receive messages out of the tunnel endpoint.  There should be a single 
+     * instance of this object per tunnel so that it can tell what tunnel various
+     * messages come in on (e.g. to prevent DataMessages arriving from anywhere 
+     * other than the client's inbound tunnels)
+     * 
+     */
+    public interface DefragmentedReceiver {
+        /**
+         * Receive a fully formed I2NPMessage out of the tunnel
+         *
+         * @param msg message received 
+         * @param toRouter where we are told to send the message (null means locally)
+         * @param toTunnel where we are told to send the message (null means locally or to the specified router)
+         */
+        public void receiveComplete(I2NPMessage msg, Hash toRouter, TunnelId toTunnel);
+    }
+    
+    private class RemoveFailed implements SimpleTimer.TimedEvent {
+        private FragmentedMessage _msg;
+        public RemoveFailed(FragmentedMessage msg) {
+            _msg = msg;
+        }
+        public void timeReached() {
+            boolean removed = false;
+            synchronized (_fragmentedMessages) {
+                removed = (null != _fragmentedMessages.remove(new Long(_msg.getMessageId())));
+            }
+            if (removed) {
+                if (_log.shouldLog(Log.WARN))
+                    _log.warn("Dropped failed fragmented message: " + _msg);
+            } else {
+                // succeeded before timeout
+            }
+        }
+        
+    }
+}

router/java/src/net/i2p/router/tunnel/FragmentTest.java

+package net.i2p.router.tunnel;
+
+import java.util.ArrayList;
+import java.util.Date;
+
+import net.i2p.I2PAppContext;
+import net.i2p.data.Base64;
+import net.i2p.data.DataHelper;
+import net.i2p.data.Hash;
+import net.i2p.data.TunnelId;
+import net.i2p.data.i2np.DataMessage;
+import net.i2p.data.i2np.I2NPMessage;
+import net.i2p.data.i2np.I2NPMessageHandler;
+import net.i2p.util.Log;
+
+/**
+ * Simple test to see if the fragmentation is working, testing the preprocessor,
+ * FragmentHandler, and FragmentedMessage operation.
+ *
+ */
+public class FragmentTest {
+    private I2PAppContext _context;
+    private Log _log;
+    
+    public FragmentTest() {
+        _context = I2PAppContext.getGlobalContext();
+        _log = _context.logManager().getLog(FragmentTest.class);
+    }
+    
+    /**
+     * Send a message that fits inside a single fragment through
+     *
+     */
+    public void runSingle() {
+        DataMessage m = new DataMessage(_context);
+        byte data[] = new byte[949];
+        _context.random().nextBytes(data);
+        m.setData(data);
+        m.setUniqueId(42);
+        m.setMessageExpiration(new Date(_context.clock().now() + 60*1000));
+        ArrayList messages = new ArrayList();
+        TunnelGateway.Pending pending = new TunnelGateway.Pending(m, null, null);
+        messages.add(pending);
+        
+        TrivialPreprocessor pre = new TrivialPreprocessor(_context);
+        SenderImpl sender = new SenderImpl();
+        FragmentHandler handler = new FragmentHandler(_context, new DefragmentedReceiverImpl(m));
+        ReceiverImpl receiver = new ReceiverImpl(handler, 0);
+        byte msg[] = m.toByteArray();
+        _log.debug("SEND(" + msg.length + "): " + Base64.encode(msg) + " " + _context.sha().calculateHash(msg).toBase64());
+        pre.preprocessQueue(messages, new SenderImpl(), receiver);
+    }
+    
+    /**
+     * Send a message with two fragments through with no delay
+     *
+     */
+    public void runMultiple() {
+        DataMessage m = new DataMessage(_context);
+        byte data[] = new byte[2048];
+        _context.random().nextBytes(data);
+        m.setData(data);
+        m.setUniqueId(42);
+        m.setMessageExpiration(new Date(_context.clock().now() + 60*1000));
+        ArrayList messages = new ArrayList();
+        TunnelGateway.Pending pending = new TunnelGateway.Pending(m, null, null);
+        messages.add(pending);
+        
+        TrivialPreprocessor pre = new TrivialPreprocessor(_context);
+        SenderImpl sender = new SenderImpl();
+        FragmentHandler handler = new FragmentHandler(_context, new DefragmentedReceiverImpl(m));
+        ReceiverImpl receiver = new ReceiverImpl(handler, 0);
+        byte msg[] = m.toByteArray();
+        _log.debug("SEND(" + msg.length + "): " + Base64.encode(msg) + " " + _context.sha().calculateHash(msg).toBase64());
+        pre.preprocessQueue(messages, new SenderImpl(), receiver);
+    }
+    
+    /**
+     * Send a fragmented message, except wait a while between each fragment, causing
+     * the defragmentation to fail (since the fragments will expire)
+     *
+     */
+    public void runDelayed() {
+        DataMessage m = new DataMessage(_context);
+        byte data[] = new byte[2048];
+        _context.random().nextBytes(data);
+        m.setData(data);
+        m.setUniqueId(42);
+        m.setMessageExpiration(new Date(_context.clock().now() + 60*1000));
+        ArrayList messages = new ArrayList();
+        TunnelGateway.Pending pending = new TunnelGateway.Pending(m, null, null);
+        messages.add(pending);
+        
+        TrivialPreprocessor pre = new TrivialPreprocessor(_context);
+        SenderImpl sender = new SenderImpl();
+        FragmentHandler handler = new FragmentHandler(_context, new DefragmentedReceiverImpl(m));
+        ReceiverImpl receiver = new ReceiverImpl(handler, 21*1000);
+        byte msg[] = m.toByteArray();
+        _log.debug("SEND(" + msg.length + "): " + Base64.encode(msg) + " " + _context.sha().calculateHash(msg).toBase64());
+        pre.preprocessQueue(messages, new SenderImpl(), receiver);
+    }
+    
+    private class SenderImpl implements TunnelGateway.Sender {
+        public void sendPreprocessed(byte[] preprocessed, TunnelGateway.Receiver receiver) {
+            receiver.receiveEncrypted(preprocessed);
+        }
+    }
+    private class ReceiverImpl implements TunnelGateway.Receiver {
+        private FragmentHandler _handler;
+        private int _delay;
+        public ReceiverImpl(FragmentHandler handler, int delay) { 
+            _handler = handler; 
+            _delay = delay;
+        }
+        public void receiveEncrypted(byte[] encrypted) {
+            _handler.receiveTunnelMessage(encrypted, 0, encrypted.length);
+            try { Thread.sleep(_delay); } catch (Exception e) {}
+        }
+    }
+    
+    private class DefragmentedReceiverImpl implements FragmentHandler.DefragmentedReceiver {
+        private I2NPMessage _expected;
+        public DefragmentedReceiverImpl(I2NPMessage expected) {
+            _expected = expected;
+        }
+        public void receiveComplete(I2NPMessage msg, Hash toRouter, TunnelId toTunnel) {
+            _log.debug("equal? " + _expected.equals(msg));
+        }
+        
+    }
+    
+    public static void main(String args[]) {
+        FragmentTest t = new FragmentTest();
+        t.runSingle();
+        t.runMultiple();
+        t.runDelayed();
+    }
+}

router/java/src/net/i2p/router/tunnel/FragmentedMessage.java

+package net.i2p.router.tunnel;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Date;
+
+import net.i2p.I2PAppContext;
+import net.i2p.data.Base64;
+import net.i2p.data.ByteArray;
+import net.i2p.data.Hash;
+import net.i2p.data.TunnelId;
+import net.i2p.data.i2np.I2NPMessage;
+import net.i2p.data.i2np.DataMessage;
+import net.i2p.data.i2np.I2NPMessageHandler;
+import net.i2p.util.Log;
+import net.i2p.util.SimpleTimer;
+
+/**
+ * Gather fragments of I2NPMessages at a tunnel endpoint, making them available 
+ * for reading when complete.
+ *
+ */
+public class FragmentedMessage {
+    private I2PAppContext _context;
+    private Log _log;
+    private long _messageId;
+    private Hash _toRouter;
+    private TunnelId _toTunnel;
+    private Map _fragments;
+    private boolean _lastReceived;
+    private int _highFragmentNum;
+    private long _createdOn;
+    private SimpleTimer.TimedEvent _expireEvent;
+    
+    public FragmentedMessage(I2PAppContext ctx) {
+        _context = ctx;
+        _log = ctx.logManager().getLog(FragmentedMessage.class);
+        _messageId = -1;
+        _toRouter = null;
+        _toTunnel = null;
+        _fragments = new HashMap(1);
+        _lastReceived = false;
+        _highFragmentNum = -1;
+        _createdOn = ctx.clock().now();
+        _expireEvent = null;
+    }
+    
+    /**
+     * Receive a followup fragment, though one of these may arrive at the endpoint
+     * prior to the fragment # 0.
+     *
+     * @param messageId what messageId is this fragment a part of 
+     * @param fragmentNum sequence number within the message (must be greater than 1)
+     * @param payload data for the fragment
+     * @param offset index into the payload where the fragment data starts (past headers/etc)
+     * @param length how much past the offset should we snag?
+     * @param isLast is this the last fragment in the message?
+     */
+    public void receive(long messageId, int fragmentNum, byte payload[], int offset, int length, boolean isLast) {
+        if (_log.shouldLog(Log.DEBUG))
+            _log.debug("Receive message " + messageId + " fragment " + fragmentNum + " with " + length + " bytes (last? " + isLast + ") offset = " + offset);
+        _messageId = messageId;
+        ByteArray ba = new ByteArray(new byte[length]);
+        System.arraycopy(payload, offset, ba.getData(), 0, length);
+        _log.debug("fragment[" + fragmentNum + "/" + offset + "/" + length + "]: " + Base64.encode(ba.getData()));
+
+        _fragments.put(new Integer(fragmentNum), ba);
+        _lastReceived = isLast;
+        if (isLast)
+            _highFragmentNum = fragmentNum;
+    }
+    
+    /**
+     * Receive the first fragment and related metadata.  This may not be the first
+     * one to arrive at the endpoint however.
+     *
+     * @param messageId what messageId is this fragment a part of 
+     * @param payload data for the fragment
+     * @param offset index into the payload where the fragment data starts (past headers/etc)
+     * @param length how much past the offset should we snag?
+     * @param isLast is this the last fragment in the message?
+     * @param toRouter what router is this destined for (may be null)
+     * @param toTunnel what tunnel is this destined for (may be null)
+     */
+    public void receive(long messageId, byte payload[], int offset, int length, boolean isLast, Hash toRouter, TunnelId toTunnel) {
+        if (_log.shouldLog(Log.DEBUG))
+            _log.debug("Receive message " + messageId + " with " + length + " bytes (last? " + isLast + ") targetting " + toRouter + " / " + toTunnel + " offset=" + offset);
+        _messageId = messageId;
+        ByteArray ba = new ByteArray(new byte[length]);
+        System.arraycopy(payload, offset, ba.getData(), 0, length);
+        _log.debug("fragment[0/" + offset + "/" + length + "]: " + Base64.encode(ba.getData()));
+        _fragments.put(new Integer(0), ba);
+        _lastReceived = isLast;
+        _toRouter = toRouter;
+        _toTunnel = toTunnel;
+        if (isLast)
+            _highFragmentNum = 0;
+    }
+    
+    public long getMessageId() { return _messageId; }
+    public Hash getTargetRouter() { return _toRouter; }
+    public TunnelId getTargetTunnel() { return _toTunnel; }
+    /** used in the fragment handler so we can cancel the expire event on success */
+    SimpleTimer.TimedEvent getExpireEvent() { return _expireEvent; }
+    void setExpireEvent(SimpleTimer.TimedEvent evt) { _expireEvent = evt; }
+    
+    /** have we received all of the fragments? */
+    public boolean isComplete() {
+        if (!_lastReceived)
+            return false;
+        for (int i = 0; i <= _highFragmentNum; i++)
+            if (!_fragments.containsKey(new Integer(i)))
+                return false;
+        return true;
+    }
+    public int getCompleteSize() {
+        if (!_lastReceived) 
+            throw new IllegalStateException("wtf, don't get the completed size when we're not complete");
+        int size = 0;
+        for (int i = 0; i <= _highFragmentNum; i++) {
+            ByteArray ba = (ByteArray)_fragments.get(new Integer(i));
+            size += ba.getData().length;
+        }
+        return size;
+    }
+    
+    /** how long has this fragmented message been alive?  */
+    public long getLifetime() { return _context.clock().now() - _createdOn; }
+    
+    
+    public void writeComplete(OutputStream out) throws IOException {
+        for (int i = 0; i <= _highFragmentNum; i++) {
+            ByteArray ba = (ByteArray)_fragments.get(new Integer(i));
+            out.write(ba.getData());
+        }
+    }
+    public void writeComplete(byte target[], int offset) {
+        for (int i = 0; i <= _highFragmentNum; i++) {
+            ByteArray ba = (ByteArray)_fragments.get(new Integer(i));
+            System.arraycopy(ba.getData(), 0, target, offset, ba.getData().length);
+            offset += ba.getData().length;
+        }
+    }
+    public byte[] toByteArray() {
+        byte rv[] = new byte[getCompleteSize()];
+        writeComplete(rv, 0);
+        return rv;
+    }
+    
+    public InputStream getInputStream() { return new FragmentInputStream(); }
+    private class FragmentInputStream extends InputStream {
+        private int _fragment;
+        private int _offset;
+        public FragmentInputStream() {
+            _fragment = 0;
+            _offset = 0;
+        }
+        public int read() throws IOException {
+            while (true) {
+                ByteArray ba = (ByteArray)_fragments.get(new Integer(_fragment));
+                if (ba == null) return -1;
+                if (_offset >= ba.getData().length) {
+                    _fragment++;
+                    _offset = 0;
+                } else {
+                    byte rv = ba.getData()[_offset];
+                    _offset++;
+                    return rv;
+                }
+            }
+        }
+    }
+    
+    public static void main(String args[]) {
+        try {
+            I2PAppContext ctx = I2PAppContext.getGlobalContext();
+            DataMessage m = new DataMessage(ctx);
+            m.setData(new byte[1024]);
+            java.util.Arrays.fill(m.getData(), (byte)0xFF);
+            m.setMessageExpiration(new Date(ctx.clock().now() + 60*1000));
+            m.setUniqueId(ctx.random().nextLong(I2NPMessage.MAX_ID_VALUE));
+            byte data[] = m.toByteArray();
+            
+            I2NPMessage r0 = new I2NPMessageHandler(ctx).readMessage(data);
+            System.out.println("peq? " + r0.equals(m));
+            
+            FragmentedMessage msg = new FragmentedMessage(ctx);
+            msg.receive(m.getUniqueId(), data, 0, 500, false, null, null);
+            msg.receive(m.getUniqueId(), 1, data, 500, 500, false);
+            msg.receive(m.getUniqueId(), 2, data, 1000, data.length-1000, true);
+            if (!msg.isComplete()) throw new RuntimeException("Not complete?");
+            
+            byte recv[] = msg.toByteArray();
+            I2NPMessage r = new I2NPMessageHandler(ctx).readMessage(recv);
+            System.out.println("eq? " + m.equals(r));
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+}

router/java/src/net/i2p/router/tunnel/HopProcessor.java

@@ -20,7 +20,9 @@ public class HopProcessor {
     private Log _log;
     protected HopConfig _config;
     private IVValidator _validator;
-    
+        
+    /** helpful flag for debugging */
+    static final boolean USE_ENCRYPTION = true;
     static final int IV_LENGTH = 16;
     
     public HopProcessor(I2PAppContext ctx, HopConfig config) {
@@ -31,6 +33,7 @@ public class HopProcessor {
     }
     
     protected IVValidator createValidator() { 
+        // yeah, we'll use an O(1) validator later (e.g. bloom filter)
         return new HashSetIVValidator();
     }
     
@@ -71,8 +74,10 @@ public class HopProcessor {
             //_log.debug("IV received: " + Base64.encode(iv));
             //_log.debug("Before:" + Base64.encode(orig, IV_LENGTH, orig.length - IV_LENGTH));
         }
-        encrypt(orig, offset, length);
-        updateIV(orig, offset);
+        if (USE_ENCRYPTION) {
+            encrypt(orig, offset, length);
+            updateIV(orig, offset);
+        }
         if (_log.shouldLog(Log.DEBUG)) {
             //_log.debug("Data after processing: " + Base64.encode(orig, IV_LENGTH, orig.length - IV_LENGTH));
             //_log.debug("IV sent: " + Base64.encode(orig, 0, IV_LENGTH));

router/java/src/net/i2p/router/tunnel/InboundEndpointProcessor.java

 package net.i2p.router.tunnel;
 
 import net.i2p.I2PAppContext;
+import net.i2p.data.Base64;
 import net.i2p.data.DataHelper;
 import net.i2p.data.Hash;
 import net.i2p.util.Log;
@@ -17,7 +18,9 @@ public class InboundEndpointProcessor {
     private I2PAppContext _context;
     private Log _log;
     private TunnelCreatorConfig _config;
-    private IVValidator _validator;
+    private IVValidator _validator;    
+    
+    static final boolean USE_ENCRYPTION = HopProcessor.USE_ENCRYPTION;
     
     public InboundEndpointProcessor(I2PAppContext ctx, TunnelCreatorConfig cfg) {
         _context = ctx;
@@ -52,7 +55,9 @@ public class InboundEndpointProcessor {
         }
         
         // inbound endpoints and outbound gateways have to undo the crypto in the same way
-        OutboundGatewayProcessor.decrypt(_context, _config, iv, orig, offset, length);
+        if (USE_ENCRYPTION)
+            OutboundGatewayProcessor.decrypt(_context, _config, iv, orig, offset, length);
+        
         return true;
     }
 }

router/java/src/net/i2p/router/tunnel/InboundGatewayProcessor.java

@@ -19,15 +19,14 @@ public class InboundGatewayProcessor extends HopProcessor {
     }
 
     /**
-     * Since we are the inbound gateway, pick a random IV, ignore the 'prev'
-     * hop, and encrypt the message like every other participant.
+     * Since we are the inbound gateway, use the IV given to us as the first 
+     * 16 bytes, ignore the 'prev' hop, and encrypt the message like every 
+     * other participant.
      *
      */
-    public boolean process(byte orig[], int offset, int length, Hash prev) {
-        byte iv[] = new byte[IV_LENGTH];
-        _context.random().nextBytes(iv);
-        System.arraycopy(iv, 0, orig, offset, IV_LENGTH);
-        
-        return super.process(orig, offset, length, null);
+    public void process(byte orig[], int offset, int length) {
+        boolean ok = super.process(orig, offset, length, null);
+        if (!ok) 
+            throw new RuntimeException("wtf, we are the gateway, how did it fail?");
     }
 }

router/java/src/net/i2p/router/tunnel/InboundGatewayTest.java

+package net.i2p.router.tunnel;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+import net.i2p.I2PAppContext;
+import net.i2p.data.Base64;
+import net.i2p.data.ByteArray;
+import net.i2p.data.DataHelper;
+import net.i2p.data.Hash;
+import net.i2p.data.TunnelId;
+import net.i2p.data.i2np.DataMessage;
+import net.i2p.data.i2np.I2NPMessage;
+import net.i2p.util.Log;
+
+/**
+ * Quick unit test for base functionality of inbound tunnel 
+ * operation
+ */
+public class InboundGatewayTest {
+    private I2PAppContext _context;
+    private Log _log;
+    private TunnelCreatorConfig _config;
+    private TunnelGateway.QueuePreprocessor _preprocessor;
+    private TunnelGateway.Sender _sender;
+    private TestReceiver _receiver;
+    private TunnelGateway _gw;
+    
+    public InboundGatewayTest() {
+        _context = I2PAppContext.getGlobalContext();
+        _log = _context.logManager().getLog(InboundGatewayTest.class);
+    }
+    
+    public void runTest() {
+        int numHops = 8;
+        int runCount = 1;
+        _config = prepareConfig(numHops);
+        _preprocessor = new TrivialPreprocessor(_context);
+        _sender = new InboundSender(_context, _config.getConfig(0));
+        _receiver = new TestReceiver(_config);
+        _gw = new TunnelGateway(_context, _preprocessor, _sender, _receiver);
+        
+        // single fragment
+        testSmall(runCount);
+        // includes target router instructions
+        testRouter(runCount);
+        // includes target router & tunnel instructions
+        testTunnel(runCount);
+        // multiple fragments
+        testLarge(runCount);
+     
+        try { Thread.sleep(5*1000); } catch (Exception e) {}
+    }
+    
+    private void testSmall(int runCount) {
+        List messages = new ArrayList(runCount);
+        long start = _context.clock().now();
+    
+        for (int i = 0; i < runCount; i++) {
+            DataMessage m = new DataMessage(_context);
+            m.setData(new byte[64]);
+            java.util.Arrays.fill(m.getData(), (byte)0xFF);
+            m.setMessageExpiration(new Date(_context.clock().now() + 60*1000));
+            m.setUniqueId(_context.random().nextLong(I2NPMessage.MAX_ID_VALUE));
+            _log.debug("Sending " + m.getUniqueId());
+            byte data[] = m.toByteArray();
+            _log.debug("SEND(" + data.length + "): " + Base64.encode(data) + " " + _context.sha().calculateHash(data).toBase64());
+            messages.add(m);
+            _gw.add(m, null, null);
+        }
+        
+        long time = _context.clock().now() - start;
+        _log.debug("Time for " + runCount + " messages: " + time);   
+        
+        List received = _receiver.clearReceived();
+        for (int i = 0; i < messages.size(); i++) {
+            if (!received.contains(((I2NPMessage)messages.get(i)))) {
+                _log.error("Message " + i + " not received");
+            }
+        }
+    }
+    
+    private void testRouter(int runCount) {
+        List messages = new ArrayList(runCount);
+        long start = _context.clock().now();
+    
+        for (int i = 0; i < runCount; i++) {
+            DataMessage m = new DataMessage(_context);
+            m.setData(new byte[64]);
+            java.util.Arrays.fill(m.getData(), (byte)0xFF);
+            m.setMessageExpiration(new Date(_context.clock().now() + 60*1000));
+            m.setUniqueId(_context.random().nextLong(I2NPMessage.MAX_ID_VALUE));
+            Hash to = new Hash(new byte[Hash.HASH_LENGTH]);
+            java.util.Arrays.fill(to.getData(), (byte)0xFF);
+            _log.debug("Sending " + m.getUniqueId() + " to " + to);
+            byte data[] = m.toByteArray();
+            _log.debug("SEND(" + data.length + "): " + Base64.encode(data) + " " + _context.sha().calculateHash(data).toBase64());
+            messages.add(m);
+            _gw.add(m, to, null);
+        }
+        
+        long time = _context.clock().now() - start;
+        _log.debug("Time for " + runCount + " messages: " + time);   
+        
+        List received = _receiver.clearReceived();
+        for (int i = 0; i < messages.size(); i++) {
+            if (!received.contains(((I2NPMessage)messages.get(i)))) {
+                _log.error("Message " + i + " not received");
+            }
+        }
+    }
+    
+    private void testTunnel(int runCount) {
+        List messages = new ArrayList(runCount);
+        long start = _context.clock().now();
+    
+        for (int i = 0; i < runCount; i++) {
+            DataMessage m = new DataMessage(_context);
+            m.setData(new byte[64]);
+            java.util.Arrays.fill(m.getData(), (byte)0xFF);
+            m.setMessageExpiration(new Date(_context.clock().now() + 60*1000));
+            m.setUniqueId(_context.random().nextLong(I2NPMessage.MAX_ID_VALUE));
+            Hash to = new Hash(new byte[Hash.HASH_LENGTH]);
+            java.util.Arrays.fill(to.getData(), (byte)0xFF);
+            TunnelId tunnel = new TunnelId(42);
+            _log.debug("Sending " + m.getUniqueId() + " to " + to + "/" + tunnel);
+            byte data[] = m.toByteArray();
+            _log.debug("SEND(" + data.length + "): " + Base64.encode(data) + " " + _context.sha().calculateHash(data).toBase64());
+            messages.add(m);
+            _gw.add(m, to, tunnel);
+        }
+        
+        long time = _context.clock().now() - start;
+        _log.debug("Time for " + runCount + " messages: " + time);   
+        
+        List received = _receiver.clearReceived();
+        for (int i = 0; i < messages.size(); i++) {
+            if (!received.contains(((I2NPMessage)messages.get(i)))) {
+                _log.error("Message " + i + " not received");
+            }
+        }
+    }
+    
+    private void testLarge(int runCount) {
+        List messages = new ArrayList(runCount);
+        long start = _context.clock().now();
+    
+        for (int i = 0; i < runCount; i++) {
+            DataMessage m = new DataMessage(_context);
+            m.setData(new byte[1024]);
+            java.util.Arrays.fill(m.getData(), (byte)0xFF);
+            m.setMessageExpiration(new Date(_context.clock().now() + 60*1000));
+            m.setUniqueId(_context.random().nextLong(I2NPMessage.MAX_ID_VALUE));
+            _log.debug("Sending " + m.getUniqueId());
+            byte data[] = m.toByteArray();
+            _log.debug("SEND(" + data.length + "): " + Base64.encode(data) + " " + _context.sha().calculateHash(data).toBase64());
+            messages.add(m);
+            _gw.add(m, null, null);
+        }
+        
+        long time = _context.clock().now() - start;
+        try { Thread.sleep(60*1000); } catch (Exception e) {}
+        _log.debug("Time for " + runCount + " messages: " + time);   
+        
+        List received = _receiver.clearReceived();
+        for (int i = 0; i < messages.size(); i++) {
+            if (!received.contains(((I2NPMessage)messages.get(i)))) {
+                _log.error("Message " + i + " not received");
+            }
+        }
+    }
+    
+    private class TestReceiver implements TunnelGateway.Receiver, FragmentHandler.DefragmentedReceiver {
+        private TunnelCreatorConfig _config;
+        private FragmentHandler _handler;
+        private List _received;
+        public TestReceiver(TunnelCreatorConfig config) {
+            _config = config;
+            _handler = new FragmentHandler(_context, TestReceiver.this);
+            _received = new ArrayList(1000);
+        }
+        public void receiveEncrypted(byte[] encrypted) {
+            // fake all the hops...
+            
+            for (int i = 1; i <= _config.getLength() - 1; i++) {
+                HopProcessor hop = new HopProcessor(_context, _config.getConfig(i));
+                boolean ok = hop.process(encrypted, 0, encrypted.length, _config.getConfig(i).getReceiveFrom());
+                if (!ok)
+                    _log.error("Error processing at hop " + i);
+                //else
+                //    _log.info("Processing OK at hop " + i);
+            }
+            
+            // now handle it at the endpoint
+            InboundEndpointProcessor end = new InboundEndpointProcessor(_context, _config);
+            boolean ok = end.retrievePreprocessedData(encrypted, 0, encrypted.length, _config.getPeer(_config.getLength()-1));
+            if (!ok)
+                _log.error("Error retrieving cleartext at the endpoint");
+            
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("Received      " + Base64.encode(encrypted));
+
+            _handler.receiveTunnelMessage(encrypted, 0, encrypted.length);
+            _log.debug("\n\ndone receiving message\n\n");
+        }        
+        public void receiveComplete(I2NPMessage msg, Hash toRouter, TunnelId toTunnel) {
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("Completed     " + msg.getUniqueId() + " to " + toRouter + "/" + toTunnel);
+            _received.add(msg);
+        }
+        public List clearReceived() { 
+            List rv = _received; 
+            _received = new ArrayList();
+            return rv;
+        }
+    }
+    
+    private TunnelCreatorConfig prepareConfig(int numHops) {
+        Hash peers[] = new Hash[numHops];
+        byte tunnelIds[][] = new byte[numHops][4];
+        for (int i = 0; i < numHops; i++) {
+            peers[i] = new Hash();
+            peers[i].setData(new byte[Hash.HASH_LENGTH]);
+            _context.random().nextBytes(peers[i].getData());
+            _context.random().nextBytes(tunnelIds[i]);
+        }
+        
+        TunnelCreatorConfig config = new TunnelCreatorConfig(numHops, false);
+        for (int i = 0; i < numHops; i++) {
+            config.setPeer(i, peers[i]);
+            HopConfig cfg = config.getConfig(i);
+            cfg.setExpiration(_context.clock().now() + 60000);
+            cfg.setIVKey(_context.keyGenerator().generateSessionKey());
+            cfg.setLayerKey(_context.keyGenerator().generateSessionKey());
+            if (i > 0)
+                cfg.setReceiveFrom(peers[i-1]);
+            else
+                cfg.setReceiveFrom(null);
+            cfg.setReceiveTunnelId(tunnelIds[i]);
+            if (i < numHops - 1) {
+                cfg.setSendTo(peers[i+1]);
+                cfg.setSendTunnelId(tunnelIds[i+1]);
+            } else {
+                cfg.setSendTo(null);
+                cfg.setSendTunnelId(null);
+            }
+        }
+        return config;
+    }
+    
+    public static void main(String args[]) {
+        InboundGatewayTest test = new InboundGatewayTest();
+        test.runTest();
+    }
+}

router/java/src/net/i2p/router/tunnel/InboundSender.java

+package net.i2p.router.tunnel;
+
+import net.i2p.I2PAppContext;
+import net.i2p.data.Base64;
+import net.i2p.util.Log;
+
+/**
+ * Receive the preprocessed data for an inbound gateway, encrypt it, and forward
+ * it on to the first hop.
+ *
+ */
+public class InboundSender implements TunnelGateway.Sender {
+    private I2PAppContext _context;
+    private Log _log;
+    private InboundGatewayProcessor _processor;
+    
+    static final boolean USE_ENCRYPTION = HopProcessor.USE_ENCRYPTION;
+    
+    public InboundSender(I2PAppContext ctx, HopConfig config) {
+        _context = ctx;
+        _log = ctx.logManager().getLog(InboundSender.class);
+        _processor = new InboundGatewayProcessor(_context, config);
+    }
+    
+    public void sendPreprocessed(byte[] preprocessed, TunnelGateway.Receiver receiver) {
+        if (USE_ENCRYPTION)
+            _processor.process(preprocessed, 0, preprocessed.length);
+        receiver.receiveEncrypted(preprocessed);
+    }
+}

router/java/src/net/i2p/router/tunnel/InboundTest.java

@@ -24,18 +24,19 @@ public class InboundTest {
         int numHops = 8;
         TunnelCreatorConfig config = prepareConfig(numHops);
         long start = _context.clock().now();
-        for (int i = 0; i < 1000; i++) 
+        for (int i = 0; i < 1; i++) 
             runTest(numHops, config);
         long time = _context.clock().now() - start;
         _log.debug("Time for 1000 messages: " + time);
     }
     
     private void runTest(int numHops, TunnelCreatorConfig config) {
-        byte orig[] = new byte[1024];
-        byte message[] = new byte[1024];
+        byte orig[] = new byte[128];
+        byte message[] = new byte[128];
         _context.random().nextBytes(orig); // might as well fill the IV
         System.arraycopy(orig, 0, message, 0, message.length);
         
+        _log.debug("orig: \n" + Base64.encode(orig, 16, orig.length-16));
         InboundGatewayProcessor p = new InboundGatewayProcessor(_context, config.getConfig(0));
         p.process(message, 0, message.length, null);
         
@@ -51,8 +52,10 @@ public class InboundTest {
         
         InboundEndpointProcessor end = new InboundEndpointProcessor(_context, config);
         boolean ok = end.retrievePreprocessedData(message, 0, message.length, config.getPeer(numHops-1));
-        if (!ok)
+        if (!ok) {
             _log.error("Error retrieving cleartext at the endpoint");
+            try { Thread.sleep(5*1000); } catch (Exception e) {}
+        }
         
         //_log.debug("After: " + Base64.encode(message, 16, orig.length-16));
         boolean eq = DataHelper.eq(orig, 16, message, 16, orig.length - 16);

router/java/src/net/i2p/router/tunnel/OutboundGatewayProcessor.java

@@ -16,7 +16,9 @@ public class OutboundGatewayProcessor {
     private I2PAppContext _context;
     private Log _log;
     private TunnelCreatorConfig _config;
-    
+        
+    static final boolean USE_ENCRYPTION = HopProcessor.USE_ENCRYPTION;
+
     public OutboundGatewayProcessor(I2PAppContext ctx, TunnelCreatorConfig cfg) {
         _context = ctx;
         _log = ctx.logManager().getLog(OutboundGatewayProcessor.class);
@@ -27,20 +29,22 @@ public class OutboundGatewayProcessor {
      * Since we are the outbound gateway, pick a random IV and wrap the preprocessed 
      * data so that it will be exposed at the endpoint.
      *
-     * @param orig original data with an extra 16 bytes prepended.
+     * @param orig original data with an extra 16 byte IV prepended.
      * @param offset index into the array where the extra 16 bytes (IV) begins
      * @param length how much of orig can we write to (must be a multiple of 16).
      */
     public void process(byte orig[], int offset, int length) {
         byte iv[] = new byte[HopProcessor.IV_LENGTH];
-        _context.random().nextBytes(iv);
-        System.arraycopy(iv, 0, orig, offset, HopProcessor.IV_LENGTH);
+        //_context.random().nextBytes(iv);
+        //System.arraycopy(iv, 0, orig, offset, HopProcessor.IV_LENGTH);
+        System.arraycopy(orig, offset, iv, 0, HopProcessor.IV_LENGTH);
         
         if (_log.shouldLog(Log.DEBUG)) {
-            _log.debug("Original random IV: " + Base64.encode(iv));
-            _log.debug("data:  " + Base64.encode(orig, iv.length, length - iv.length));
+            //_log.debug("Original random IV: " + Base64.encode(iv));
+            //_log.debug("data:  " + Base64.encode(orig, iv.length, length - iv.length));
         }
-        decrypt(_context, _config, iv, orig, offset, length);
+        if (USE_ENCRYPTION)
+            decrypt(_context, _config, iv, orig, offset, length);
     }
     
     /**
@@ -55,8 +59,8 @@ public class OutboundGatewayProcessor {
         for (int i = cfg.getLength()-1; i >= 0; i--) {
             decrypt(ctx, iv, orig, offset, length, cur, cfg.getConfig(i));
             if (log.shouldLog(Log.DEBUG)) {
-                //_log.debug("IV at hop " + i + ": " + Base64.encode(orig, offset, iv.length));
-                //log.debug("hop " + i + ": " + Base64.encode(orig, offset + iv.length, length - iv.length));
+                //log.debug("IV at hop " + i + ": " + Base64.encode(orig, offset, HopProcessor.IV_LENGTH));
+                //log.debug("hop " + i + ": " + Base64.encode(orig, offset + HopProcessor.IV_LENGTH, length - HopProcessor.IV_LENGTH));
             }
         }
     }

router/java/src/net/i2p/router/tunnel/TrivialPreprocessor.java

+package net.i2p.router.tunnel;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import net.i2p.I2PAppContext;
+import net.i2p.data.Base64;
+import net.i2p.data.DataHelper;
+import net.i2p.data.Hash;
+import net.i2p.data.i2np.I2NPMessage;
+import net.i2p.util.Log;
+
+/** 
+ * Do the simplest thing possible for preprocessing - for each message available,
+ * turn it into the minimum number of fragmented preprocessed blocks, sending 
+ * each of those out.  This does not coallesce message fragments or delay for more
+ * optimal throughput.
+ *
+ */
+public class TrivialPreprocessor implements TunnelGateway.QueuePreprocessor {
+    private I2PAppContext _context;
+    private Log _log;
+    
+    private static final int PREPROCESSED_SIZE = 1024;
+    private static final int IV_SIZE = HopProcessor.IV_LENGTH;
+    
+    public TrivialPreprocessor(I2PAppContext ctx) {
+        _context = ctx;
+        _log = ctx.logManager().getLog(TrivialPreprocessor.class);
+    }
+    
+    public boolean preprocessQueue(List pending, TunnelGateway.Sender sender, TunnelGateway.Receiver rec) {
+        while (pending.size() > 0) {
+            TunnelGateway.Pending msg = (TunnelGateway.Pending)pending.remove(0);
+            byte preprocessed[][] = preprocess(msg);
+            for (int i = 0; i < preprocessed.length; i++) {
+                if (_log.shouldLog(Log.DEBUG))
+                    _log.debug("Preprocessed: " + Base64.encode(preprocessed[i]));
+                sender.sendPreprocessed(preprocessed[i], rec);
+            }
+        }
+        return false;
+    }
+    
+    private byte[][] preprocess(TunnelGateway.Pending msg) {
+        List fragments = new ArrayList(1);
+
+        while (msg.getOffset() < msg.getData().length) {
+            fragments.add(preprocessFragment(msg));
+            _log.debug("\n\nafter preprocessing fragment\n\n");
+        }
+
+        byte rv[][] = new byte[fragments.size()][];
+        for (int i = 0; i < fragments.size(); i++)
+            rv[i] = (byte[])fragments.get(i);
+        return rv;
+    }
+    
+    /**
+     * Preprocess the next available fragment off the given one in phases:
+     * First, write it out as { instructions + payload + random IV }, calculate the 
+     * SHA256 of that, then move the instructions + payload to the end 
+     * of the target, setting IV as the beginning.  Then add the necessary random pad
+     * bytes after the IV, followed by the first 4 bytes of that SHA256, lining up
+     * exactly to meet the beginning of the instructions. (i hope)
+     *
+     */
+    private byte[] preprocessFragment(TunnelGateway.Pending msg) {
+        if (msg.getOffset() <= 0)
+            return preprocessFirstFragment(msg);
+        else
+            return preprocessSubsequentFragment(msg);
+    }
+    
+    
+    /** is this a follw up byte? */
+    private static final byte MASK_IS_SUBSEQUENT = FragmentHandler.MASK_IS_SUBSEQUENT;
+    /** how should this be delivered?  shift this 5 the right and get TYPE_* */
+    private static final byte MASK_TYPE = FragmentHandler.MASK_TYPE;
+    /** is this the first of a fragmented message? */
+    private static final byte MASK_FRAGMENTED = FragmentHandler.MASK_FRAGMENTED;
+    /** are there follow up headers? */
+    private static final byte MASK_EXTENDED = FragmentHandler.MASK_EXTENDED;
+    private static final byte MASK_TUNNEL = (byte)(FragmentHandler.TYPE_TUNNEL << 5);
+    private static final byte MASK_ROUTER = (byte)(FragmentHandler.TYPE_ROUTER << 5);
+    
+    private byte[] preprocessFirstFragment(TunnelGateway.Pending msg) {
+        boolean fragmented = false;
+        byte iv[] = new byte[IV_SIZE];
+        _context.random().nextBytes(iv);
+        
+        byte target[] = new byte[PREPROCESSED_SIZE];
+        
+        int instructionsLength = getInstructionsSize(msg);
+        int payloadLength = msg.getData().length;
+        if (payloadLength + instructionsLength + IV_SIZE + 1 + 4 > PREPROCESSED_SIZE) {
+            fragmented = true;
+            instructionsLength += 4; // messageId
+            payloadLength = PREPROCESSED_SIZE - IV_SIZE - 1 - 4 - instructionsLength;
+        }
+        
+        int offset = 0;
+        
+        // first fragment, or full message
+        target[offset] = 0x0;
+        if (msg.getToTunnel() != null)
+            target[offset] |= MASK_TUNNEL;
+        else if (msg.getToRouter() != null) 
+            target[offset] |= MASK_ROUTER;
+        if (fragmented)
+            target[offset] |= MASK_FRAGMENTED;
+        
+        _log.debug("CONTROL: " + Integer.toHexString(target[offset]));
+
+        offset++;
+
+        if (msg.getToTunnel() != null) {
+            DataHelper.toLong(target, offset, 4, msg.getToTunnel().getTunnelId());
+            offset += 4;
+        }
+        if (msg.getToRouter() != null) {
+            System.arraycopy(msg.getToRouter().getData(), 0, target, offset, Hash.HASH_LENGTH);
+            offset += Hash.HASH_LENGTH;
+        }
+        if (fragmented) {
+            DataHelper.toLong(target, offset, 4, msg.getMessageId());
+            _log.debug("writing messageId= " + msg.getMessageId() + " at offset " + offset);
+            offset += 4;
+        }
+        DataHelper.toLong(target, offset, 2, payloadLength);
+        offset += 2;
+        //_log.debug("raw data    : " + Base64.encode(msg.getData()));
+        System.arraycopy(msg.getData(), 0, target, offset, payloadLength);
+        _log.debug("fragment[" + msg.getFragmentNumber()+ "/" + (PREPROCESSED_SIZE - offset - payloadLength) + "/" + payloadLength + "]: " + Base64.encode(target, offset, payloadLength));
+
+        offset += payloadLength;
+        
+        // payload ready, now H(instructions+payload+IV)
+        System.arraycopy(iv, 0, target, offset, IV_SIZE);
+        Hash h = _context.sha().calculateHash(target, 0, offset + IV_SIZE);
+        _log.debug("before shift: " + Base64.encode(target));
+        // now shiiiiiift
+        int distance = PREPROCESSED_SIZE - offset;
+        System.arraycopy(target, 0, target, distance, offset);
+        
+        _log.debug("fragments begin at " + distance + " (size=" + payloadLength + " offset=" + offset +")");
+        
+        java.util.Arrays.fill(target, 0, distance, (byte)0x0);
+        _log.debug("after shift: " + Base64.encode(target));
+        
+        offset = 0;
+        System.arraycopy(iv, 0, target, offset, IV_SIZE);
+        offset += IV_SIZE;
+        System.arraycopy(h.getData(), 0, target, offset, 4);
+        offset += 4;
+        //_log.debug("before pad  : " + Base64.encode(target));
+        
+        if (!fragmented) {
+            // fits in a single message, so may be smaller than the full size
+           int numPadBytes = PREPROCESSED_SIZE     // max 
+                             - IV_SIZE             // hmm..
+                             - 4                   // 4 bytes of the SHA256
+                             - 1                   // the 0x00 after the padding
+                             - payloadLength       // the, er, payload
+                             - instructionsLength; // wanna guess?
+
+           //_log.debug("# pad bytes:  " + numPadBytes + " payloadLength: " + payloadLength + " instructions: " + instructionsLength);
+           
+           for (int i = 0; i < numPadBytes; i++) {
+               if (false) {
+                   target[offset] = 0x0;
+                   offset++;
+               } else { 
+                   // wouldn't it be nice if random could write to an array?
+                   byte rnd = (byte)_context.random().nextInt();
+                   if (rnd != 0x0) {
+                       target[offset] = rnd;
+                       offset++;
+                   } else {
+                       i--;
+                   }
+               }
+           }
+        }
+        target[offset] = 0x0; // no padding here
+        offset++;
+        
+        msg.setOffset(payloadLength);
+        msg.incrementFragmentNumber();
+        return target;
+    }
+    
+    private byte[] preprocessSubsequentFragment(TunnelGateway.Pending msg) {
+        boolean isLast = true;
+        byte iv[] = new byte[IV_SIZE];
+        _context.random().nextBytes(iv);
+        
+        byte target[] = new byte[PREPROCESSED_SIZE];
+        
+        int instructionsLength = getInstructionsSize(msg);
+        int payloadLength = msg.getData().length - msg.getOffset();
+        if (payloadLength + instructionsLength + IV_SIZE + 1 + 4 > PREPROCESSED_SIZE) {
+            isLast = false;
+            payloadLength = PREPROCESSED_SIZE - IV_SIZE - 1 - 4 - instructionsLength;
+        }
+        
+        int offset = 0;
+        
+        // first fragment, or full message
+        target[offset] = 0x0;
+        target[offset] |= MASK_IS_SUBSEQUENT;
+        target[offset] |= (byte)(msg.getFragmentNumber() << 1); // max 63 fragments
+        if (isLast)
+            target[offset] |= 1;  
+        
+        _log.debug("CONTROL: " + Integer.toHexString((int)target[offset]) + "/" + Base64.encode(target, offset, 1) + " at offset " + offset);
+
+        offset++;
+
+        DataHelper.toLong(target, offset, 4, msg.getMessageId());
+        offset += 4;
+        DataHelper.toLong(target, offset, 2, payloadLength);
+        offset += 2;
+        System.arraycopy(msg.getData(), msg.getOffset(), target, offset, payloadLength);
+        _log.debug("fragment[" + msg.getFragmentNumber()+ "/" + offset + "/" + payloadLength + "]: " + Base64.encode(target, offset, payloadLength));
+
+        offset += payloadLength;
+
+        // payload ready, now H(instructions+payload+IV)
+        System.arraycopy(iv, 0, target, offset, IV_SIZE);
+        Hash h = _context.sha().calculateHash(target, 0, offset + IV_SIZE);
+        // now shiiiiiift
+        int distance = PREPROCESSED_SIZE - offset;
+        System.arraycopy(target, 0, target, distance, offset);
+                        
+        _log.debug("fragments begin at " + distance + " (size=" + payloadLength + " offset=" + offset +")");
+
+        offset = 0;
+        System.arraycopy(iv, 0, target, 0, IV_SIZE);
+        offset += IV_SIZE;
+        
+        System.arraycopy(h.getData(), 0, target, offset, 4);
+        offset += 4;
+        
+        if (isLast) {
+            // this is the last message, so may be smaller than the full size
+           int numPadBytes = PREPROCESSED_SIZE     // max 
+                             - IV_SIZE             // hmm..
+                             - 4                   // 4 bytes of the SHA256
+                             - 1                   // the 0x00 after the padding
+                             - payloadLength       // the, er, payload
+                             - instructionsLength; // wanna guess?
+
+           for (int i = 0; i < numPadBytes; i++) {
+               // wouldn't it be nice if random could write to an array?
+               byte rnd = (byte)_context.random().nextInt();
+               if (rnd != 0x0) {
+                   target[offset] = rnd;
+                   offset++;
+               } else {
+                   i--;
+               }
+           }
+
+           _log.debug("# pad bytes: " + numPadBytes);
+        }
+        target[offset] = 0x0; // end of padding
+        offset++;
+        
+        msg.setOffset(msg.getOffset() + payloadLength);
+        msg.incrementFragmentNumber();
+        return target;
+    }
+    
+    private int getInstructionsSize(TunnelGateway.Pending msg) {
+        if (msg.getFragmentNumber() > 0) 
+            return 7;
+        int header = 1;
+        if (msg.getToTunnel() != null)
+            header += 4;
+        if (msg.getToRouter() != null)
+            header += 32;
+        header += 2;
+        return header;
+    }
+    
+}

router/java/src/net/i2p/router/tunnel/TunnelGateway.java

+package net.i2p.router.tunnel;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import net.i2p.I2PAppContext;
+import net.i2p.data.Hash;
+import net.i2p.data.TunnelId;
+import net.i2p.data.i2np.I2NPMessage;
+import net.i2p.util.Log;
+import net.i2p.util.SimpleTimer;
+
+/**
+ * Serve as the gatekeeper for a tunnel, accepting messages, coallescing and/or
+ * fragmenting them before wrapping them up for tunnel delivery. The flow here
+ * is: <ol>
+ * <li>add an I2NPMessage (and a target tunnel/router, if necessary)</li>
+ * <li>that message is queued up into a TunnelGateway.Pending and offered to the
+ *     assigned QueuePreprocessor.</li>
+ * <li>that QueuePreprocessor may then take off any of the TunnelGateway.Pending
+ *     messages or instruct the TunnelGateway to offer it the messages again in
+ *     a short while (in an attempt to coallesce them).
+ * <li>when the QueueProcessor accepts a TunnelGateway.Pending, it preprocesses
+ *     it into fragments, forwarding each preprocessed fragment group through 
+ *     the Sender.</li>
+ * <li>the Sender then encrypts the preprocessed data and delivers it to the 
+ *     Receiver.</li>
+ * <li>the Receiver now has the encrypted message and may do with it as it 
+ *     pleases (e.g. wrap it as necessary and enqueue it onto the OutNetMessagePool,
+ *     or if debugging, verify that it can be decrypted properly)</li>
+ * </ol>
+ *
+ */
+public class TunnelGateway {
+    private I2PAppContext _context;
+    private Log _log;
+    private List _queue;
+    private QueuePreprocessor _preprocessor;
+    private Sender _sender;
+    private Receiver _receiver;
+    private long _lastFlush;
+    private int _flushFrequency;
+    private DelayedFlush _delayedFlush;
+    
+    /**
+     * @param preprocessor this pulls Pending messages off a list, builds some
+     *                     full preprocessed messages, and pumps those into the sender
+     * @param sender this takes a preprocessed message, encrypts it, and sends it to 
+     *               the receiver
+     * @param receiver this receives the encrypted message and forwards it off 
+     *                 to the first hop
+     */
+    public TunnelGateway(I2PAppContext context, QueuePreprocessor preprocessor, Sender sender, Receiver receiver) {
+        _context = context;
+        _log = context.logManager().getLog(TunnelGateway.class);
+        _queue = new ArrayList(4);
+        _preprocessor = preprocessor;
+        _sender = sender;
+        _receiver = receiver;
+        _flushFrequency = 500;
+        _delayedFlush = new DelayedFlush();
+        _lastFlush = _context.clock().now();
+    }
+    
+    /**
+     * Add a message to be sent down the tunnel, either sending it now (perhaps
+     * coallesced with other pending messages) or after a brief pause (_flushFrequency).
+     * If it is queued up past its expiration, it is silently dropped
+     *
+     * @param msg message to be sent through the tunnel
+     * @param toRouter router to send to after the endpoint (or null for endpoint processing)
+     * @param toTunnel tunnel to send to after the endpoint (or null for endpoint or router processing)
+     */
+    public void add(I2NPMessage msg, Hash toRouter, TunnelId toTunnel) {
+        boolean delayedFlush = false;
+        
+        Pending cur = new Pending(msg, toRouter, toTunnel);
+        synchronized (_queue) {
+            _queue.add(cur);
+            delayedFlush = _preprocessor.preprocessQueue(_queue, _sender, _receiver);
+            _lastFlush = _context.clock().now();
+            
+            // expire any as necessary, even if its framented
+            for (int i = 0; i < _queue.size(); i++) {
+                Pending m = (Pending)_queue.get(i);
+                if (m.getExpiration() < _lastFlush) {
+                    _queue.remove(i);
+                    i--;
+                }
+            }
+        }
+        
+        if (delayedFlush) {
+            SimpleTimer.getInstance().addEvent(_delayedFlush, _flushFrequency);
+        }
+    }
+    
+    public interface Sender {
+        /**
+         * Take the preprocessed data containing zero or more fragments, encrypt
+         * it, and pass it on to the receiver
+         *
+         * @param preprocessed IV + (rand padding) + 0x0 + Hash[0:3] + {instruction+fragment}*
+         */
+        public void sendPreprocessed(byte preprocessed[], Receiver receiver);
+    }
+        
+    public interface QueuePreprocessor {
+        /** 
+         * @param pending list of Pending objects for messages either unsent
+         *                or partly sent.  This list should be update with any
+         *                values removed (the preprocessor owns the lock)
+         * @return true if we should delay before preprocessing again 
+         */
+        public boolean preprocessQueue(List pending, Sender sender, Receiver receiver);
+    }
+    
+    public interface Receiver {
+        /**
+         * Take the encrypted data and send it off to the next hop
+         */
+        public void receiveEncrypted(byte encrypted[]);
+    }
+    
+    public static class Pending {
+        private Hash _toRouter;
+        private TunnelId _toTunnel;
+        private long _messageId;
+        private long _expiration;
+        private byte _remaining[];
+        private int _offset;
+        private int _fragmentNumber;
+        
+        public Pending(I2NPMessage message, Hash toRouter, TunnelId toTunnel) {
+            _toRouter = toRouter;
+            _toTunnel = toTunnel;
+            _messageId = message.getUniqueId();
+            _expiration = message.getMessageExpiration().getTime();
+            _remaining = message.toByteArray();
+            _offset = 0;
+            _fragmentNumber = 0;
+        }
+        /** may be null */
+        public Hash getToRouter() { return _toRouter; }
+        /** may be null */
+        public TunnelId getToTunnel() { return _toTunnel; }
+        public long getMessageId() { return _messageId; }
+        public long getExpiration() { return _expiration; }
+        /** raw unfragmented message to send */
+        public byte[] getData() { return _remaining; }
+        /** index into the data to be sent */
+        public int getOffset() { return _offset; }
+        /** move the offset */
+        public void setOffset(int offset) { _offset = offset; }
+        /** which fragment are we working on (0 for the first fragment) */
+        public int getFragmentNumber() { return _fragmentNumber; }
+        /** ok, fragment sent, increment what the next will be */
+        public void incrementFragmentNumber() { _fragmentNumber++; }
+    }
+    
+    private class DelayedFlush implements SimpleTimer.TimedEvent {
+        public void timeReached() {
+            long now = _context.clock().now();
+            synchronized (_queue) {
+                if ( (_queue.size() > 0) && (_lastFlush + _flushFrequency < now) ) {
+                    _preprocessor.preprocessQueue(_queue, _sender, _receiver);
+                    _lastFlush = _context.clock().now();
+                }
+            }
+        }
+    }
+}