From 312534b635e5df2f5ac728aa5bdf24b4465eb318 Mon Sep 17 00:00:00 2001
From: zzz <zzz@mail.i2p>
Date: Sat, 26 Mar 2011 05:27:34 +0000
Subject: [PATCH] - Change disk format to add magic number to all pages -
Change blockfile magic number to reflect new format - Cleanups and javadocs
---
.../net/metanotion/io/block/BlockFile.java | 49 ++++++++++++++-----
.../io/block/index/BSkipLevels.java | 7 +++
.../metanotion/io/block/index/BSkipList.java | 14 ++++++
.../metanotion/io/block/index/BSkipSpan.java | 46 +++++++++++++++--
.../metanotion/io/block/index/IBSkipSpan.java | 14 ++++--
5 files changed, 109 insertions(+), 21 deletions(-)
diff --git a/core/java/src/net/metanotion/io/block/BlockFile.java b/core/java/src/net/metanotion/io/block/BlockFile.java
index 867617bdb8..2d08ec0015 100644
--- a/core/java/src/net/metanotion/io/block/BlockFile.java
+++ b/core/java/src/net/metanotion/io/block/BlockFile.java
@@ -51,10 +51,20 @@ import net.metanotion.util.skiplist.SkipList;
import net.i2p.I2PAppContext;
import net.i2p.util.Log;
-class CorruptFileException extends IOException { }
-class BadFileFormatException extends IOException { }
-class BadVersionException extends IOException { }
-
+/**
+ * On-disk format:
+ * Magic number (6 bytes)
+ * Version major/minor (2 bytes)
+ * file length (long)
+ * free list start (unsigned int)
+ * is mounted (unsigned short) 0 = no, 1 = yes
+ * span size (unsigned short)
+ *
+ * Metaindex skiplist is on page 2
+ *
+ * Pages are 1 KB and are numbered starting from 1.
+ * e.g. the Metaindex skiplist is at offset 1024 bytes
+ */
public class BlockFile {
public static final long PAGESIZE = 1024;
public static final long OFFSET_MOUNTED = 20;
@@ -62,7 +72,13 @@ public class BlockFile {
public RandomAccessInterface file;
- private long magicBytes = 0x3141deadbeef0100L;
+ private static final int MAJOR = 0x01;
+ private static final int MINOR = 0x01;
+ // I2P changed magic number, format changed, magic numbers now on all pages
+ private static final long MAGIC_BASE = 0x3141de4932500000L; // 0x3141de I 2 P 00 00
+ private static final long MAGIC = MAGIC_BASE | (MAJOR << 8) | MINOR;
+ private long magicBytes = MAGIC;
+ public static final int MAGIC_CONT = 0x434f4e54; // "CONT"
private long fileLen = PAGESIZE * 2;
private int freeListStart = 0;
private int mounted = 0;
@@ -123,14 +139,17 @@ public class BlockFile {
if(curNextPage==0) {
curNextPage = this.allocPage();
BlockFile.pageSeek(this.file, curNextPage);
+ this.file.writeInt(MAGIC_CONT);
this.file.writeInt(0);
BlockFile.pageSeek(this.file, curPage);
+ this.file.skipBytes(4); // skip magic
this.file.writeInt(curNextPage);
}
BlockFile.pageSeek(this.file, curNextPage);
curPage = curNextPage;
+ this.file.skipBytes(4); // skip magic
curNextPage = this.file.readUnsignedInt();
- pageCounter = 4;
+ pageCounter = 8;
len = ((int) BlockFile.PAGESIZE) - pageCounter;
}
this.file.write(data, dct, Math.min(len, data.length - dct));
@@ -152,9 +171,12 @@ public class BlockFile {
int len = ((int) BlockFile.PAGESIZE) - pageCounter;
if(len <= 0) {
BlockFile.pageSeek(this.file, curNextPage);
+ int magic = this.file.readInt();
+ if (magic != MAGIC_CONT)
+ throw new IOException("Bad SkipSpan continuation magic number 0x" + Integer.toHexString(magic) + " on page " + curNextPage);
curPage = curNextPage;
curNextPage = this.file.readUnsignedInt();
- pageCounter = 4;
+ pageCounter = 8;
len = ((int) BlockFile.PAGESIZE) - pageCounter;
}
res = this.file.read(arr, dct, Math.min(len, arr.length - dct));
@@ -184,16 +206,19 @@ public class BlockFile {
}
readSuperBlock();
- if(magicBytes != 0x3141deadbeef0100L) {
- if((magicBytes & 0x3141deadbeef0000L) == 0x3141deadbeef0000L) {
- throw new BadVersionException();
+ if(magicBytes != MAGIC) {
+ if((magicBytes & MAGIC_BASE) == MAGIC_BASE) {
+ throw new IOException("Expected " + MAJOR + '.' + MINOR +
+ " but got " + (magicBytes >> 8 & 0xff) + '.' + (magicBytes & 0xff));
} else {
- throw new BadFileFormatException();
+ throw new IOException("Bad magic number");
}
}
if (mounted != 0)
log.warn("Warning - file was not previously closed");
- if(fileLen != file.length()) { throw new CorruptFileException(); }
+ if(fileLen != file.length())
+ throw new IOException("Expected file length " + fileLen +
+ " but actually " + file.length());
mount();
metaIndex = new BSkipList(spanSize, this, 2, new StringBytes(), new IntBytes());
diff --git a/core/java/src/net/metanotion/io/block/index/BSkipLevels.java b/core/java/src/net/metanotion/io/block/index/BSkipLevels.java
index 7450422134..5f17929d65 100644
--- a/core/java/src/net/metanotion/io/block/index/BSkipLevels.java
+++ b/core/java/src/net/metanotion/io/block/index/BSkipLevels.java
@@ -38,12 +38,14 @@ import net.metanotion.util.skiplist.SkipSpan;
/**
* On-disk format:
+ * Magic number (long)
* max height (unsigned short)
* non-null height (unsigned short)
* span page (unsigned int)
* height number of level pages (unsigned ints)
*/
public class BSkipLevels extends SkipLevels {
+ private static final long MAGIC = 0x42534c6576656c73l; // "BSLevels"
public final int levelPage;
public final int spanPage;
public final BlockFile bf;
@@ -53,6 +55,9 @@ public class BSkipLevels extends SkipLevels {
this.bf = bf;
BlockFile.pageSeek(bf.file, levelPage);
+ long magic = bf.file.readLong();
+ if (magic != MAGIC)
+ throw new IOException("Bad SkipLevels magic number 0x" + Long.toHexString(magic) + " on page " + levelPage);
bsl.levelHash.put(new Integer(this.levelPage), this);
@@ -87,6 +92,7 @@ public class BSkipLevels extends SkipLevels {
public static void init(BlockFile bf, int page, int spanPage, int maxHeight) throws IOException {
BlockFile.pageSeek(bf.file, page);
+ bf.file.writeLong(MAGIC);
bf.file.writeShort((short) maxHeight);
bf.file.writeShort(0);
bf.file.writeInt(spanPage);
@@ -95,6 +101,7 @@ public class BSkipLevels extends SkipLevels {
public void flush() {
try {
BlockFile.pageSeek(bf.file, levelPage);
+ bf.file.writeLong(MAGIC);
bf.file.writeShort((short) levels.length);
int i = 0;
for( ; i < levels.length; i++) {
diff --git a/core/java/src/net/metanotion/io/block/index/BSkipList.java b/core/java/src/net/metanotion/io/block/index/BSkipList.java
index e28165a9d0..8a8a409508 100644
--- a/core/java/src/net/metanotion/io/block/index/BSkipList.java
+++ b/core/java/src/net/metanotion/io/block/index/BSkipList.java
@@ -37,7 +37,16 @@ import net.metanotion.io.Serializer;
import net.metanotion.io.block.BlockFile;
import net.metanotion.util.skiplist.*;
+/**
+ * On-disk format:
+ * Magic number (long)
+ * first span page (unsigned int)
+ * first level page (unsigned int)
+ * size (unsigned int)
+ * spans (unsigned int)
+ */
public class BSkipList extends SkipList {
+ private static final long MAGIC = 0x536b69704c697374l; // "SkipList"
public int firstSpanPage = 0;
public int firstLevelPage = 0;
public int skipPage = 0;
@@ -59,6 +68,9 @@ public class BSkipList extends SkipList {
this.bf = bf;
BlockFile.pageSeek(bf.file, skipPage);
+ long magic = bf.file.readLong();
+ if (magic != MAGIC)
+ throw new IOException("Bad SkipList magic number 0x" + Long.toHexString(magic) + " on page " + skipPage);
firstSpanPage = bf.file.readUnsignedInt();
firstLevelPage = bf.file.readUnsignedInt();
size = bf.file.readUnsignedInt();
@@ -84,6 +96,7 @@ public class BSkipList extends SkipList {
public void flush() {
try {
BlockFile.pageSeek(bf.file, skipPage);
+ bf.file.writeLong(MAGIC);
bf.file.writeInt(firstSpanPage);
bf.file.writeInt(firstLevelPage);
bf.file.writeInt(size);
@@ -114,6 +127,7 @@ public class BSkipList extends SkipList {
int firstSpan = bf.allocPage();
int firstLevel = bf.allocPage();
BlockFile.pageSeek(bf.file, page);
+ bf.file.writeLong(MAGIC);
bf.file.writeInt(firstSpan);
bf.file.writeInt(firstLevel);
bf.file.writeInt(0);
diff --git a/core/java/src/net/metanotion/io/block/index/BSkipSpan.java b/core/java/src/net/metanotion/io/block/index/BSkipSpan.java
index 8e9dc19d12..2188a6db30 100644
--- a/core/java/src/net/metanotion/io/block/index/BSkipSpan.java
+++ b/core/java/src/net/metanotion/io/block/index/BSkipSpan.java
@@ -36,8 +36,30 @@ import net.metanotion.io.block.BlockFile;
import net.metanotion.util.skiplist.SkipList;
import net.metanotion.util.skiplist.SkipSpan;
+/**
+ * On-disk format:
+ *
+ * First Page:
+ * Magic number (int)
+ * overflow page (unsigned int)
+ * previous page (unsigned int)
+ * next page (unsigned int)
+ * max keys (unsigned short)
+ * number of keys (unsigned short)
+ * for each key:
+ * key length (unsigned short)
+ * value length (unsigned short)
+ * key data
+ * value data
+ *
+ * Overflow pages:
+ * Magic number (int)
+ * next overflow page (unsigned int)
+ */
public class BSkipSpan extends SkipSpan {
-
+ protected static final int MAGIC = 0x5370616e; // "Span"
+ protected static final int HEADER_LEN = 20;
+ protected static final int CONT_HEADER_LEN = 8;
protected BlockFile bf;
protected int page;
protected int overflowPage;
@@ -52,6 +74,7 @@ public class BSkipSpan extends SkipSpan {
public static void init(BlockFile bf, int page, int spanSize) throws IOException {
BlockFile.pageSeek(bf.file, page);
+ bf.file.writeInt(MAGIC);
bf.file.writeInt(0);
bf.file.writeInt(0);
bf.file.writeInt(0);
@@ -73,6 +96,7 @@ public class BSkipSpan extends SkipSpan {
int next;
while(curPage != 0) {
BlockFile.pageSeek(bf.file, curPage);
+ bf.file.skipBytes(4); // skip magic
next = bf.file.readUnsignedInt();
bf.freePage(curPage);
curPage = next;
@@ -91,6 +115,7 @@ public class BSkipSpan extends SkipSpan {
private void fflush() {
try {
BlockFile.pageSeek(bf.file, page);
+ bf.file.writeInt(MAGIC);
bf.file.writeInt(overflowPage);
bf.file.writeInt((prev != null) ? ((BSkipSpan) prev).page : 0);
bf.file.writeInt((next != null) ? ((BSkipSpan) next).page : 0);
@@ -102,7 +127,7 @@ public class BSkipSpan extends SkipSpan {
int[] curNextPage = new int[1];
curNextPage[0] = this.overflowPage;
int[] pageCounter = new int[1];
- pageCounter[0] = 16;
+ pageCounter[0] = HEADER_LEN;
byte[] keyData;
byte[] valData;
@@ -111,14 +136,17 @@ public class BSkipSpan extends SkipSpan {
if(curNextPage[0] == 0) {
curNextPage[0] = bf.allocPage();
BlockFile.pageSeek(bf.file, curNextPage[0]);
+ bf.file.writeInt(BlockFile.MAGIC_CONT);
bf.file.writeInt(0);
BlockFile.pageSeek(bf.file, curPage);
+ bf.file.skipBytes(4); // skip magic
bf.file.writeInt(curNextPage[0]);
}
BlockFile.pageSeek(bf.file, curNextPage[0]);
curPage = curNextPage[0];
+ bf.file.skipBytes(4); // skip magic
curNextPage[0] = bf.file.readUnsignedInt();
- pageCounter[0] = 4;
+ pageCounter[0] = CONT_HEADER_LEN;
}
// Drop bad entry without throwing exception
if (keys[i] == null || vals[i] == null) {
@@ -144,7 +172,9 @@ public class BSkipSpan extends SkipSpan {
curPage = bf.writeMultiPageData(keyData, curPage, pageCounter, curNextPage);
curPage = bf.writeMultiPageData(valData, curPage, pageCounter, curNextPage);
}
+ // FIXME why seek and rescan the overflow page?
BlockFile.pageSeek(bf.file, this.page);
+ bf.file.skipBytes(4); // skip magic
this.overflowPage = bf.file.readUnsignedInt();
} catch (IOException ioe) { throw new RuntimeException("Error writing to database", ioe); }
// FIXME can't get there from here
@@ -171,6 +201,9 @@ public class BSkipSpan extends SkipSpan {
BlockFile.pageSeek(bf.file, spanPage);
+ int magic = bf.file.readInt();
+ if (magic != MAGIC)
+ throw new IOException("Bad SkipSpan magic number 0x" + Integer.toHexString(magic) + " on page " + spanPage);
bss.overflowPage = bf.file.readUnsignedInt();
bss.prevPage = bf.file.readUnsignedInt();
bss.nextPage = bf.file.readUnsignedInt();
@@ -200,15 +233,18 @@ public class BSkipSpan extends SkipSpan {
int[] curNextPage = new int[1];
curNextPage[0] = this.overflowPage;
int[] pageCounter = new int[1];
- pageCounter[0] = 16;
+ pageCounter[0] = HEADER_LEN;
// System.out.println("Span Load " + sz + " nKeys " + nKeys + " page " + curPage);
int fail = 0;
for(int i=0;i<this.nKeys;i++) {
if((pageCounter[0] + 4) > BlockFile.PAGESIZE) {
BlockFile.pageSeek(this.bf.file, curNextPage[0]);
curPage = curNextPage[0];
+ int magic = bf.file.readInt();
+ if (magic != BlockFile.MAGIC_CONT)
+ throw new IOException("Bad SkipSpan magic number 0x" + Integer.toHexString(magic) + " on page " + curPage);
curNextPage[0] = this.bf.file.readUnsignedInt();
- pageCounter[0] = 4;
+ pageCounter[0] = CONT_HEADER_LEN;
}
ksz = this.bf.file.readUnsignedShort();
vsz = this.bf.file.readUnsignedShort();
diff --git a/core/java/src/net/metanotion/io/block/index/IBSkipSpan.java b/core/java/src/net/metanotion/io/block/index/IBSkipSpan.java
index 47e8775dab..4591b4b6bf 100644
--- a/core/java/src/net/metanotion/io/block/index/IBSkipSpan.java
+++ b/core/java/src/net/metanotion/io/block/index/IBSkipSpan.java
@@ -114,7 +114,7 @@ public class IBSkipSpan extends BSkipSpan {
int[] curNextPage = new int[1];
curNextPage[0] = this.overflowPage;
int[] pageCounter = new int[1];
- pageCounter[0] = 16;
+ pageCounter[0] = HEADER_LEN;
ksz = this.bf.file.readUnsignedShort();
this.bf.file.skipBytes(2); //vsz
pageCounter[0] +=4;
@@ -134,8 +134,11 @@ public class IBSkipSpan extends BSkipSpan {
*/
private void seekData() throws IOException {
BlockFile.pageSeek(this.bf.file, this.page);
+ int magic = bf.file.readInt();
+ if (magic != MAGIC)
+ throw new IOException("Bad SkipSpan magic number 0x" + Integer.toHexString(magic) + " on page " + this.page);
// 3 ints and 2 shorts
- this.bf.file.skipBytes(16);
+ this.bf.file.skipBytes(HEADER_LEN - 4);
}
/**
@@ -157,15 +160,18 @@ public class IBSkipSpan extends BSkipSpan {
int[] curNextPage = new int[1];
curNextPage[0] = this.overflowPage;
int[] pageCounter = new int[1];
- pageCounter[0] = 16;
+ pageCounter[0] = HEADER_LEN;
int fail = 0;
//System.out.println("Span Load " + sz + " nKeys " + nKeys + " page " + curPage);
for(int i=0;i<this.nKeys;i++) {
if((pageCounter[0] + 4) > BlockFile.PAGESIZE) {
BlockFile.pageSeek(this.bf.file, curNextPage[0]);
curPage = curNextPage[0];
+ int magic = bf.file.readInt();
+ if (magic != BlockFile.MAGIC_CONT)
+ throw new IOException("Bad SkipSpan continuation magic number 0x" + Integer.toHexString(magic) + " on page " + curPage);
curNextPage[0] = this.bf.file.readUnsignedInt();
- pageCounter[0] = 4;
+ pageCounter[0] = CONT_HEADER_LEN;
}
ksz = this.bf.file.readUnsignedShort();
vsz = this.bf.file.readUnsignedShort();
--
GitLab