diff --git a/core/java/src/net/i2p/client/naming/BlockfileNamingService.java b/core/java/src/net/i2p/client/naming/BlockfileNamingService.java index 8c7962fc3..712b29082 100644 --- a/core/java/src/net/i2p/client/naming/BlockfileNamingService.java +++ b/core/java/src/net/i2p/client/naming/BlockfileNamingService.java @@ -32,6 +32,7 @@ import net.i2p.util.Log; import net.metanotion.io.Serializer; import net.metanotion.io.block.BlockFile; +import net.metanotion.util.skiplist.SkipIterator; import net.metanotion.util.skiplist.SkipList; @@ -361,6 +362,33 @@ public class BlockfileNamingService extends DummyNamingService { return d; } + private void dumpDB() { + synchronized(_bf) { + if (_isClosed) + _log.error("Database is closed"); + for (String list : _lists) { + try { + SkipList sl = _bf.getIndex(list, _stringSerializer, _destSerializer); + if (sl == null) { + _log.error("No list found for " + list); + continue; + } + int i = 0; + for (SkipIterator iter = sl.iterator(); iter.hasNext(); ) { + String key = (String) iter.nextKey(); + DestEntry de = (DestEntry) iter.next(); + _log.error("DB " + list + " key " + key + " val " + de); + i++; + } + _log.error(i + " entries found for " + list); + } catch (IOException ioe) { + _log.error("Fail", ioe); + break; + } + } + } + } + public void close() { synchronized(_bf) { try { @@ -506,6 +534,7 @@ public class BlockfileNamingService extends DummyNamingService { } System.out.println("BFNS took " + DataHelper.formatDuration(System.currentTimeMillis() - start)); System.out.println("found " + found + " notfound " + notfound); + bns.dumpDB(); bns.close(); HostsTxtNamingService htns = new HostsTxtNamingService(I2PAppContext.getGlobalContext()); diff --git a/core/java/src/net/metanotion/io/block/index/BSkipList.java b/core/java/src/net/metanotion/io/block/index/BSkipList.java index e93f8260f..e7c8741e0 100644 --- a/core/java/src/net/metanotion/io/block/index/BSkipList.java +++ b/core/java/src/net/metanotion/io/block/index/BSkipList.java @@ -46,7 +46,7 @@ public class BSkipList extends SkipList { public HashMap spanHash = new HashMap(); public HashMap levelHash = new HashMap(); - protected BSkipList() { } + private final boolean fileOnly; public BSkipList(int spanSize, BlockFile bf, int skipPage, Serializer key, Serializer val) throws IOException { this(spanSize, bf, skipPage, key, val, false); @@ -65,6 +65,7 @@ public class BSkipList extends SkipList { spans = bf.file.readInt(); //System.out.println(size + " " + spans); + this.fileOnly = fileOnly; if (fileOnly) first = new IBSkipSpan(bf, this, firstSpanPage, key, val); else @@ -127,4 +128,34 @@ public class BSkipList extends SkipList { return (max > cells) ? cells : max; } + @Override + public SkipIterator iterator() { + if (!this.fileOnly) + return super.iterator(); + return new IBSkipIterator(first, 0); + } + + @Override + public SkipIterator min() { + return iterator(); + } + + @Override + public SkipIterator max() { + if (!this.fileOnly) + return super.max(); + SkipSpan ss = stack.getEnd(); + return new IBSkipIterator(ss, ss.nKeys - 1); + } + + @Override + public SkipIterator find(Comparable key) { + if (!this.fileOnly) + return super.find(key); + int[] search = new int[1]; + SkipSpan ss = stack.getSpan(stack.levels.length - 1, key, search); + if(search[0] < 0) { search[0] = -1 * (search[0] + 1); } + return new IBSkipIterator(ss, search[0]); + } + } diff --git a/core/java/src/net/metanotion/io/block/index/BSkipSpan.java b/core/java/src/net/metanotion/io/block/index/BSkipSpan.java index 9ebabd9fa..9ade78b02 100644 --- a/core/java/src/net/metanotion/io/block/index/BSkipSpan.java +++ b/core/java/src/net/metanotion/io/block/index/BSkipSpan.java @@ -129,7 +129,7 @@ public class BSkipSpan extends SkipSpan { private static void load(BSkipSpan bss, BlockFile bf, BSkipList bsl, int spanPage, Serializer key, Serializer val) throws IOException { loadInit(bss, bf, bsl, spanPage, key, val); - loadData(bss, bf, spanPage, key, val); + bss.loadData(); } /** @@ -157,34 +157,34 @@ public class BSkipSpan extends SkipSpan { * I2P - second half of load() * Load the whole span's keys and values into memory */ - protected static void loadData(BSkipSpan bss, BlockFile bf, int spanPage, Serializer key, Serializer val) throws IOException { - bss.keys = new Comparable[bss.spanSize]; - bss.vals = new Object[bss.spanSize]; + protected void loadData() throws IOException { + this.keys = new Comparable[this.spanSize]; + this.vals = new Object[this.spanSize]; int ksz, vsz; - int curPage = spanPage; + int curPage = this.page; int[] curNextPage = new int[1]; - curNextPage[0] = bss.overflowPage; + curNextPage[0] = this.overflowPage; int[] pageCounter = new int[1]; pageCounter[0] = 16; // System.out.println("Span Load " + sz + " nKeys " + nKeys + " page " + curPage); - for(int i=0;i BlockFile.PAGESIZE) { - BlockFile.pageSeek(bf.file, curNextPage[0]); + BlockFile.pageSeek(this.bf.file, curNextPage[0]); curPage = curNextPage[0]; - curNextPage[0] = bf.file.readInt(); + curNextPage[0] = this.bf.file.readInt(); pageCounter[0] = 4; } - ksz = bf.file.readShort(); - vsz = bf.file.readShort(); + ksz = this.bf.file.readShort(); + vsz = this.bf.file.readShort(); pageCounter[0] +=4; byte[] k = new byte[ksz]; byte[] v = new byte[vsz]; - curPage = bf.readMultiPageData(k, curPage, pageCounter, curNextPage); - curPage = bf.readMultiPageData(v, curPage, pageCounter, curNextPage); + curPage = this.bf.readMultiPageData(k, curPage, pageCounter, curNextPage); + curPage = this.bf.readMultiPageData(v, curPage, pageCounter, curNextPage); // System.out.println("i=" + i + ", Page " + curPage + ", offset " + pageCounter[0] + " ksz " + ksz + " vsz " + vsz); - bss.keys[i] = (Comparable) bss.keySer.construct(k); - bss.vals[i] = bss.valSer.construct(v); + this.keys[i] = (Comparable) this.keySer.construct(k); + this.vals[i] = this.valSer.construct(v); } } diff --git a/core/java/src/net/metanotion/io/block/index/IBSkipIterator.java b/core/java/src/net/metanotion/io/block/index/IBSkipIterator.java new file mode 100644 index 000000000..a43df79c8 --- /dev/null +++ b/core/java/src/net/metanotion/io/block/index/IBSkipIterator.java @@ -0,0 +1,137 @@ +/* +Copyright (c) 2006, Matthew Estes +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + * Neither the name of Metanotion Software nor the names of its +contributors may be used to endorse or promote products derived from this +software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +package net.metanotion.io.block.index; + +import java.io.IOException; +import java.util.NoSuchElementException; + +import net.metanotion.util.skiplist.SkipIterator; +import net.metanotion.util.skiplist.SkipSpan; + +/** + I2P + Overridden to load the span when required and null out the keys and values + when the iterator leaves the span. + If the caller does not iterate all the way through, the last span + will remain in memory. +*/ +public class IBSkipIterator extends SkipIterator { + + public IBSkipIterator(SkipSpan ss, int index) { + super(ss, index); + } + + /** + * @return the next value, and advances the index + * @throws NoSuchElementException + * @throws RuntimeException on IOE + */ + @Override + public Object next() { + Object o; + if(index < ss.nKeys) { + if (ss.vals == null) { + try { + ((IBSkipSpan)ss).seekAndLoadData(); + } catch (IOException ioe) { + throw new RuntimeException("Error in iterator", ioe); + } + } + o = ss.vals[index]; + } else { + throw new NoSuchElementException(); + } + + if(index < (ss.nKeys-1)) { + index++; + } else if(ss.next != null) { + ss.keys = null; + ss.vals = null; + ss = ss.next; + index = 0; + } else { + ss.keys = null; + ss.vals = null; + index = ss.nKeys; + } + return o; + } + + /** + * The key. Does NOT advance the index. + * @return the key for which the value will be returned in the subsequent call to next() + * @throws NoSuchElementException + * @throws RuntimeException on IOE + */ + @Override + public Comparable nextKey() { + Comparable c; + if(index < ss.nKeys) { + if (ss.keys == null) { + try { + ((IBSkipSpan)ss).seekAndLoadData(); + } catch (IOException ioe) { + throw new RuntimeException("Error in iterator", ioe); + } + } + return ss.keys[index]; + } + throw new NoSuchElementException(); + } + + /** + * @return the previous value, and decrements the index + * @throws NoSuchElementException + * @throws RuntimeException on IOE + */ + @Override + public Object previous() { + if(index > 0) { + index--; + } else if(ss.prev != null) { + ss.keys = null; + ss.vals = null; + ss = ss.prev; + if(ss.nKeys <= 0) { throw new NoSuchElementException(); } + index = (ss.nKeys - 1); + } else { + ss.keys = null; + ss.vals = null; + throw new NoSuchElementException(); + } + if (ss.vals == null) { + try { + ((IBSkipSpan)ss).seekAndLoadData(); + } catch (IOException ioe) { + throw new RuntimeException("Error in iterator", ioe); + } + } + return ss.vals[index]; + } +} diff --git a/core/java/src/net/metanotion/io/block/index/IBSkipSpan.java b/core/java/src/net/metanotion/io/block/index/IBSkipSpan.java index 07838390f..315b180de 100644 --- a/core/java/src/net/metanotion/io/block/index/IBSkipSpan.java +++ b/core/java/src/net/metanotion/io/block/index/IBSkipSpan.java @@ -92,12 +92,13 @@ public class IBSkipSpan extends BSkipSpan { * I2P - second half of load() * Load the whole span's keys and values into memory */ - protected static void loadData(IBSkipSpan bss, BlockFile bf, int spanPage, Serializer key, Serializer val) throws IOException { - BSkipSpan.loadData(bss, bf, spanPage, key, val); - if (bss.nKeys > 0) - bss.firstKey = bss.keys[0]; + @Override + protected void loadData() throws IOException { + super.loadData(); + if (this.nKeys > 0) + this.firstKey = this.keys[0]; if (DEBUG) - System.err.println("Loaded data for page " + spanPage + " containing " + bss.nKeys + '/' + bss.spanSize + " first key: " + bss.firstKey); + System.err.println("Loaded data for page " + this.page + " containing " + this.nKeys + '/' + this.spanSize + " first key: " + this.firstKey); } /** @@ -134,10 +135,11 @@ public class IBSkipSpan extends BSkipSpan { /** * Seek to the start of the span and load the data + * Package private so BSkipIterator can call it */ - private void seekAndLoadData() throws IOException { + void seekAndLoadData() throws IOException { seekData(); - loadData(this, this.bf, this.page, this.keySer, this.valSer); + loadData(); } /** diff --git a/core/java/src/net/metanotion/util/skiplist/SkipIterator.java b/core/java/src/net/metanotion/util/skiplist/SkipIterator.java index cf2d6e067..bc723050c 100644 --- a/core/java/src/net/metanotion/util/skiplist/SkipIterator.java +++ b/core/java/src/net/metanotion/util/skiplist/SkipIterator.java @@ -35,10 +35,13 @@ import java.util.NoSuchElementException; This is not a complete ListIterator, in particular, since the skip list is a map and is therefore indexed by Comparable objects instead of int's, the nextIndex and previousIndex methods are not really relevant. + + To be clear, this is an iterator through the values. + To get the key, call nextKey() BEFORE calling next(). */ public class SkipIterator implements ListIterator { - SkipSpan ss; - int index; + protected SkipSpan ss; + protected int index; protected SkipIterator() { } public SkipIterator(SkipSpan ss, int index) { @@ -52,6 +55,10 @@ public class SkipIterator implements ListIterator { return false; } + /** + * @return the next value, and advances the index + * @throws NoSuchElementException + */ public Object next() { Object o; if(index < ss.nKeys) { @@ -71,6 +78,11 @@ public class SkipIterator implements ListIterator { return o; } + /** + * The key. Does NOT advance the index. + * @return the key for which the value will be returned in the subsequent call to next() + * @throws NoSuchElementException + */ public Comparable nextKey() { Comparable c; if(index < ss.nKeys) { return ss.keys[index]; } @@ -83,6 +95,10 @@ public class SkipIterator implements ListIterator { return false; } + /** + * @return the previous value, and decrements the index + * @throws NoSuchElementException + */ public Object previous() { if(index > 0) { index--; diff --git a/core/java/src/net/metanotion/util/skiplist/SkipList.java b/core/java/src/net/metanotion/util/skiplist/SkipList.java index 11216ab62..66982982b 100644 --- a/core/java/src/net/metanotion/util/skiplist/SkipList.java +++ b/core/java/src/net/metanotion/util/skiplist/SkipList.java @@ -136,6 +136,7 @@ public class SkipList { return new SkipIterator(ss, ss.nKeys - 1); } + /** @return an iterator where nextKey() is the first one greater than or equal to 'key' */ public SkipIterator find(Comparable key) { int[] search = new int[1]; SkipSpan ss = stack.getSpan(stack.levels.length - 1, key, search); @@ -163,6 +164,7 @@ public class SkipList { Repeat, with splits induced, and collapse induced. */ +/***** public static void main(String args[]) { SkipList sl = new SkipList(3); sl.put(".1", "1"); @@ -171,11 +173,14 @@ public class SkipList { sl.put(".1", "1-1"); sl.put(".2", "2"); sl.put(".3", "3"); +*****/ /* System.out.println("\n#1"); sl.print(); */ +/***** sl.put(".4", "4"); +*****/ /* System.out.println("\n#2"); sl.print(); @@ -193,6 +198,7 @@ public class SkipList { System.out.println("\n#3"); sl.print(); */ +/****** sl.put(".1", "1-2"); sl.put(".2", "2-1"); sl.put(".3", "3-1"); @@ -311,4 +317,5 @@ public class SkipList { System.out.println("Iterator: " + si.next()); } } +*****/ }