diff --git a/core/java/src/net/metanotion/io/block/index/BSkipList.java b/core/java/src/net/metanotion/io/block/index/BSkipList.java
index c510107f7aeeeac4251192402013645a1932a275..c55b850d950953a3f43456ee62a206a856cf9ca5 100644
--- a/core/java/src/net/metanotion/io/block/index/BSkipList.java
+++ b/core/java/src/net/metanotion/io/block/index/BSkipList.java
@@ -82,8 +82,8 @@ public class BSkipList extends SkipList {
firstSpanPage = bf.file.readUnsignedInt();
firstLevelPage = bf.file.readUnsignedInt();
size = bf.file.readUnsignedInt();
- spans = bf.file.readUnsignedInt();
- levelCount = bf.file.readUnsignedInt();
+ int spans = bf.file.readInt();
+ int levelCount = bf.file.readInt();
//System.out.println(size + " " + spans);
this.fileOnly = fileOnly;
@@ -92,8 +92,18 @@ public class BSkipList extends SkipList {
else
first = new BSkipSpan(bf, this, firstSpanPage, key, val);
stack = new BSkipLevels(bf, firstLevelPage, this);
+ int total = 0;
+ for (BSkipSpan ss : spanHash.values()) {
+ total += ss.nKeys;
+ }
if (BlockFile.log.shouldLog(Log.DEBUG))
- BlockFile.log.debug("Loaded " + this + " cached " + levelHash.size() + " levels and " + spanHash.size() + " spans");
+ BlockFile.log.debug("Loaded " + this + " cached " + levelHash.size() + " levels and " + spanHash.size() + " spans with " + total + " entries");
+ if (levelCount != levelHash.size() || spans != spanHash.size() || size != total) {
+ if (BlockFile.log.shouldLog(Log.WARN))
+ BlockFile.log.warn("On-disk counts were " + levelCount + " / " + spans + " / " + size + ", correcting");
+ size = total;
+ flush();
+ }
//rng = new Random(System.currentTimeMillis());
}
@@ -117,8 +127,8 @@ public class BSkipList extends SkipList {
bf.file.writeInt(firstSpanPage);
bf.file.writeInt(firstLevelPage);
bf.file.writeInt(Math.max(0, size));
- bf.file.writeInt(Math.max(0, spans));
- bf.file.writeInt(Math.max(0, levelCount));
+ bf.file.writeInt(spanHash.size());
+ bf.file.writeInt(levelHash.size());
} catch (IOException ioe) { throw new RuntimeException("Error writing to database", ioe); }
}
@@ -164,11 +174,21 @@ public class BSkipList extends SkipList {
BSkipLevels.init(bf, firstLevel, firstSpan, 4);
}
+ /**
+ * @return log2(span count), minimum 4
+ */
@Override
public int maxLevels() {
- int max = super.maxLevels();
- int cells = (BlockFile.PAGESIZE - BSkipLevels.HEADER_LEN) / 4;
- return Math.min(cells, max);
+ int hob = 0;
+ int s = spanHash.size();
+ while(s > 0) {
+ hob++;
+ s /= P;
+ }
+ int max = Math.max(hob, super.maxLevels());
+ // 252
+ //int cells = (BlockFile.PAGESIZE - BSkipLevels.HEADER_LEN) / 4;
+ return Math.min(BSkipLevels.MAX_SIZE, max);
}
@Override
@@ -203,8 +223,8 @@ public class BSkipList extends SkipList {
public void bslck(boolean isMeta, boolean fix) {
BlockFile.log.warn(" size " + this.size);
- BlockFile.log.warn(" spans " + this.spans);
- BlockFile.log.warn(" levels " + this.levelCount);
+ BlockFile.log.warn(" spans " + this.spanHash.size());
+ BlockFile.log.warn(" levels " + this.levelHash.size());
BlockFile.log.warn(" skipPage " + this.skipPage);
BlockFile.log.warn(" firstSpanPage " + this.firstSpanPage);
BlockFile.log.warn(" firstLevelPage " + this.firstLevelPage);
diff --git a/core/java/src/net/metanotion/io/block/index/BSkipSpan.java b/core/java/src/net/metanotion/io/block/index/BSkipSpan.java
index e63edc10958661a332cc90f5eaa8664f714cb052..992ed9d2e447ea9d6663d1cb433aab7342535b09 100644
--- a/core/java/src/net/metanotion/io/block/index/BSkipSpan.java
+++ b/core/java/src/net/metanotion/io/block/index/BSkipSpan.java
@@ -161,6 +161,8 @@ public class BSkipSpan extends SkipSpan {
return;
bf.file.writeShort((short) keys.length);
bf.file.writeShort((short) nKeys);
+ if (nKeys <= 0 && prev != null)
+ BlockFile.log.error("Flushing with no entries?" + this, new Exception());
int ksz, vsz;
int curPage = this.page;
@@ -264,8 +266,11 @@ public class BSkipSpan extends SkipSpan {
bss.nextPage = bf.file.readUnsignedInt();
bss.spanSize = bf.file.readUnsignedShort();
bss.nKeys = bf.file.readUnsignedShort();
- if(bss.spanSize < 1 || bss.spanSize > SkipSpan.MAX_SIZE || bss.nKeys > bss.spanSize)
- throw new IOException("Invalid span size " + bss.nKeys + " / "+ bss.spanSize);
+ if(bss.spanSize < 1 || bss.spanSize > SkipSpan.MAX_SIZE || bss.nKeys > bss.spanSize) {
+ BlockFile.log.error("Invalid span size " + bss.nKeys + " / "+ bss.spanSize);
+ bss.nKeys = 0;
+ bss.spanSize = bf.spanSize;
+ }
}
/**
@@ -298,10 +303,13 @@ public class BSkipSpan extends SkipSpan {
for(int i=0;i<this.nKeys;i++) {
if((pageCounter[0] + 4) > BlockFile.PAGESIZE) {
BlockFile.pageSeek(this.bf.file, curNextPage[0]);
- curPage = curNextPage[0];
int magic = bf.file.readInt();
- if (magic != BlockFile.MAGIC_CONT)
- throw new IOException("Bad SkipSpan magic number 0x" + Integer.toHexString(magic) + " on page " + curPage);
+ if (magic != BlockFile.MAGIC_CONT) {
+ BlockFile.log.error("Lost " + (this.nKeys - i) + " entries - Bad SkipSpan magic number 0x" + Integer.toHexString(magic) + " on page " + curNextPage[0]);
+ lostEntries(i, curPage);
+ break;
+ }
+ curPage = curNextPage[0];
curNextPage[0] = this.bf.file.readUnsignedInt();
pageCounter[0] = CONT_HEADER_LEN;
}
@@ -310,8 +318,15 @@ public class BSkipSpan extends SkipSpan {
pageCounter[0] +=4;
byte[] k = new byte[ksz];
byte[] v = new byte[vsz];
- curPage = this.bf.readMultiPageData(k, curPage, pageCounter, curNextPage);
- curPage = this.bf.readMultiPageData(v, curPage, pageCounter, curNextPage);
+ int lastGood = curPage;
+ try {
+ curPage = this.bf.readMultiPageData(k, curPage, pageCounter, curNextPage);
+ curPage = this.bf.readMultiPageData(v, curPage, pageCounter, curNextPage);
+ } catch (IOException ioe) {
+ BlockFile.log.error("Lost " + (this.nKeys - i) + " entries - Error loading " + this + " on page " + curPage, ioe);
+ lostEntries(i, lastGood);
+ break;
+ }
// System.out.println("i=" + i + ", Page " + curPage + ", offset " + pageCounter[0] + " ksz " + ksz + " vsz " + vsz);
this.keys[i] = (Comparable) this.keySer.construct(k);
this.vals[i] = this.valSer.construct(v);
@@ -336,6 +351,33 @@ public class BSkipSpan extends SkipSpan {
}
}
+ /**
+ * Attempt to recover from corrupt data in this span.
+ * All entries starting with firstBadEntry are lost.
+ * Zero out the overflow page on lastGoodPage,
+ * and corect the number of entries in the first page.
+ * We don't attempt to free the lost continuation pages.
+ */
+ protected void lostEntries(int firstBadEntry, int lastGoodPage) {
+ try {
+ this.nKeys = firstBadEntry;
+ // zero overflow page pointer
+ BlockFile.pageSeek(this.bf.file, lastGoodPage);
+ bf.file.skipBytes(4); // skip magic
+ bf.file.writeInt(0);
+ // write new number of keys
+ if (lastGoodPage != this.page) {
+ BlockFile.pageSeek(this.bf.file, this.page);
+ bf.file.skipBytes(18);
+ } else {
+ bf.file.skipBytes(10);
+ }
+ bf.file.writeShort(this.nKeys);
+ } catch (IOException ioe) {
+ BlockFile.log.error("Error while recovering from corruption of " + this, ioe);
+ }
+ }
+
protected BSkipSpan(BlockFile bf, BSkipList bsl) {
this.bf = bf;
this.bsl = bsl;
diff --git a/core/java/src/net/metanotion/io/block/index/IBSkipSpan.java b/core/java/src/net/metanotion/io/block/index/IBSkipSpan.java
index fa74cbcd2f8318c9c70ab4627dd70597190a0821..be3a44772ef8aec9f5ab3d74a55635e0528eacac 100644
--- a/core/java/src/net/metanotion/io/block/index/IBSkipSpan.java
+++ b/core/java/src/net/metanotion/io/block/index/IBSkipSpan.java
@@ -172,10 +172,13 @@ public class IBSkipSpan extends BSkipSpan {
for(int i=0;i<this.nKeys;i++) {
if((pageCounter[0] + 4) > BlockFile.PAGESIZE) {
BlockFile.pageSeek(this.bf.file, curNextPage[0]);
- curPage = curNextPage[0];
int magic = bf.file.readInt();
- if (magic != BlockFile.MAGIC_CONT)
- throw new IOException("Bad SkipSpan continuation magic number 0x" + Integer.toHexString(magic) + " on page " + curPage);
+ if (magic != BlockFile.MAGIC_CONT) {
+ BlockFile.log.error("Lost " + (this.nKeys - i) + " entries - Bad SkipSpan magic number 0x" + Integer.toHexString(magic) + " on page " + curNextPage[0]);
+ lostEntries(i, curPage);
+ break;
+ }
+ curPage = curNextPage[0];
curNextPage[0] = this.bf.file.readUnsignedInt();
pageCounter[0] = CONT_HEADER_LEN;
}
@@ -183,7 +186,13 @@ public class IBSkipSpan extends BSkipSpan {
int vsz = this.bf.file.readUnsignedShort();
pageCounter[0] +=4;
byte[] k = new byte[ksz];
- curPage = this.bf.readMultiPageData(k, curPage, pageCounter, curNextPage);
+ try {
+ curPage = this.bf.readMultiPageData(k, curPage, pageCounter, curNextPage);
+ } catch (IOException ioe) {
+ BlockFile.log.error("Lost " + (this.nKeys - i) + " entries - Error loading " + this + " on page " + curPage, ioe);
+ lostEntries(i, curPage);
+ break;
+ }
//System.out.println("i=" + i + ", Page " + curPage + ", offset " + pageCounter[0] + " ksz " + ksz + " vsz " + vsz);
Comparable ckey = (Comparable) this.keySer.construct(k);
if (ckey == null) {
@@ -197,7 +206,13 @@ public class IBSkipSpan extends BSkipSpan {
if (diff == 0) {
//System.err.println("Found " + key + " at " + i + " (first: " + this.firstKey + ')');
byte[] v = new byte[vsz];
- curPage = this.bf.readMultiPageData(v, curPage, pageCounter, curNextPage);
+ try {
+ curPage = this.bf.readMultiPageData(v, curPage, pageCounter, curNextPage);
+ } catch (IOException ioe) {
+ BlockFile.log.error("Lost " + (this.nKeys - i) + " entries - Error loading " + this + " on page " + curPage, ioe);
+ lostEntries(i, curPage);
+ break;
+ }
Object rv = this.valSer.construct(v);
if (rv == null) {
BlockFile.log.error("Null deserialized value in entry " + i + " page " + curPage +
@@ -224,6 +239,7 @@ public class IBSkipSpan extends BSkipSpan {
}
private void repair(int fail) {
+ /***** needs work
try {
loadData(false);
if (this.nKeys > 0)
@@ -233,6 +249,7 @@ public class IBSkipSpan extends BSkipSpan {
} catch (IOException ioe) {
BlockFile.log.error("Failed to repair corruption of " + fail + " entries", ioe);
}
+ *****/
}
private IBSkipSpan(BlockFile bf, BSkipList bsl) {
diff --git a/core/java/src/net/metanotion/util/skiplist/SkipLevels.java b/core/java/src/net/metanotion/util/skiplist/SkipLevels.java
index f2af9a4eb462aea8e4b9b33cdd9da227f4ac409e..8e090c48962b0a61ce56f5f41a2289ee4c8f0ee2 100644
--- a/core/java/src/net/metanotion/util/skiplist/SkipLevels.java
+++ b/core/java/src/net/metanotion/util/skiplist/SkipLevels.java
@@ -187,7 +187,6 @@ public class SkipLevels {
}
res[1] = null;
}
- sl.delSpan(res[1] != null);
}
if((bottom.nKeys == 0) && (sl.first != bottom)) { this.killInstance(); }
return res;
@@ -244,7 +243,6 @@ public class SkipLevels {
if(levels.length < height)
return slvls;
}
- sl.addSpan(height != 0);
}
return null;
}
diff --git a/core/java/src/net/metanotion/util/skiplist/SkipList.java b/core/java/src/net/metanotion/util/skiplist/SkipList.java
index 132e4e8f7f2bf3a768b0b2774724bda5c170fb0c..d2d2a26ca340a64fe9c6ead8e42ee69b95e9cfdb 100644
--- a/core/java/src/net/metanotion/util/skiplist/SkipList.java
+++ b/core/java/src/net/metanotion/util/skiplist/SkipList.java
@@ -36,7 +36,7 @@ import net.i2p.util.RandomSource;
public class SkipList {
/** the probability of each next higher level */
- private static final int P = 2;
+ protected static final int P = 2;
private static final int MIN_SLOTS = 4;
// these two are really final
protected SkipSpan first;
@@ -45,8 +45,6 @@ public class SkipList {
public static final Random rng = RandomSource.getInstance();
protected int size;
- protected int spans;
- protected int levelCount;
public void flush() { }
protected SkipList() { }
@@ -60,8 +58,6 @@ public class SkipList {
throw new IllegalArgumentException("Invalid span size");
first = new SkipSpan(span);
stack = new SkipLevels(1, first);
- spans = 1;
- levelCount = 1;
//rng = new Random(System.currentTimeMillis());
}
@@ -76,29 +72,12 @@ public class SkipList {
size--;
}
- public void addSpan(boolean addLevel) {
- spans++;
- if (addLevel)
- levelCount++;
- }
-
- public void delSpan(boolean delLevel) {
- if (spans > 0)
- spans--;
- if (delLevel && levelCount > 0)
- levelCount--;
- }
-
/**
- * @return log2(spans), minimum 4
+ * @return 4 since we don't track span count here any more - see override
+ * Fix if for some reason you want a huge in-memory skiplist.
*/
public int maxLevels() {
- int hob = 0, s = spans;
- while(s > 0) {
- hob++;
- s /= P;
- }
- return Math.max(hob, MIN_SLOTS);
+ return MIN_SLOTS;
}
/**
@@ -157,13 +136,13 @@ public class SkipList {
/** dumps all the skip levels */
public void printSL() {
- System.out.println("List size " + size + " spans " + spans);
+ System.out.println("List size " + size);
System.out.println(stack.printAll());
}
/** dumps all the data */
public void print() {
- System.out.println("List size " + size + " spans " + spans);
+ System.out.println("List size " + size);
System.out.println(first.print());
}