On Monday 26 May 2008 10:34, j16sdiz at freenetproject.org wrote:
> Author: j16sdiz
> Date: 2008-05-26 09:34:39 +0000 (Mon, 26 May 2008)
> New Revision: 20097
>
> Modified:
> branches/saltedhashstore/freenet/src/freenet/node/Node.java
>
branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
> Log:
> Data store migration
Would it make sense to move a bunch of keys from the end, truncate the old
files, and repeat until everything is moved, to minimize disk space usage
during migration? Of course this relies on a filesystem supporting sparse
files, and will likely result in the new store files being rather
fragmented...?
>
> Modified: branches/saltedhashstore/freenet/src/freenet/node/Node.java
> ===================================================================
> --- branches/saltedhashstore/freenet/src/freenet/node/Node.java
> 2008-05-26
09:34:05 UTC (rev 20096)
> +++ branches/saltedhashstore/freenet/src/freenet/node/Node.java
> 2008-05-26
09:34:39 UTC (rev 20097)
> @@ -1331,35 +1331,61 @@
> Logger.normal(this, "Initializing CHK
> Datastore");
> System.out.println("Initializing CHK Datastore
> (" + maxStoreKeys + "
keys)");
> chkDatastore = new CHKStore();
> - SaltedHashFreenetStore.construct(storeDir,
> "CHK-store", chkDatastore,
random, maxStoreKeys,
> - shutdownHook);
> + SaltedHashFreenetStore chkDataFS =
SaltedHashFreenetStore.construct(storeDir, "CHK-store",
> + chkDatastore, random, maxStoreKeys,
> shutdownHook);
> Logger.normal(this, "Initializing CHK
> Datacache");
> System.out.println("Initializing CHK Datacache
> (" + maxCacheKeys + ':'
+ maxCacheKeys + " keys)");
> chkDatacache = new CHKStore();
> - SaltedHashFreenetStore.construct(storeDir,
> "CHK-cache", chkDatacache,
random, maxCacheKeys,
> - shutdownHook);
> + SaltedHashFreenetStore chkCacheFS =
SaltedHashFreenetStore.construct(storeDir, "CHK-cache",
> + chkDatacache, random, maxCacheKeys,
> shutdownHook);
>
> Logger.normal(this, "Initializing pubKey
> Datastore");
> System.out.println("Initializing pubKey
> Datastore");
> pubKeyDatastore = new PubkeyStore();
> - SaltedHashFreenetStore.construct(storeDir,
> "PUBKEY-store",
pubKeyDatastore, random, maxStoreKeys,
> - shutdownHook);
> + SaltedHashFreenetStore pubkeyDataFS =
SaltedHashFreenetStore.construct(storeDir, "PUBKEY-store",
> + pubKeyDatastore, random, maxStoreKeys,
> shutdownHook);
> Logger.normal(this, "Initializing pubKey
> Datacache");
> System.out.println("Initializing pubKey
> Datacache (" + maxCacheKeys + "
keys)");
> pubKeyDatacache = new PubkeyStore();
> - SaltedHashFreenetStore.construct(storeDir,
> "PUBKEY-cache",
pubKeyDatacache, random, maxCacheKeys,
> - shutdownHook);
> + SaltedHashFreenetStore pubkeyCacheFS =
SaltedHashFreenetStore.construct(storeDir, "PUBKEY-cache",
> + pubKeyDatacache, random, maxCacheKeys,
> shutdownHook);
>
> Logger.normal(this, "Initializing SSK
> Datastore");
> System.out.println("Initializing SSK
> Datastore");
> sskDatastore = new SSKStore(this);
> - SaltedHashFreenetStore.construct(storeDir,
> "SSK-store", sskDatastore,
random, maxStoreKeys,
> - shutdownHook);
> + SaltedHashFreenetStore sskDataFS =
SaltedHashFreenetStore.construct(storeDir, "SSK-store",
> + sskDatastore, random, maxStoreKeys,
> shutdownHook);
> Logger.normal(this, "Initializing SSK
> Datacache");
> System.out.println("Initializing SSK Datacache
> (" + maxCacheKeys + "
keys)");
> sskDatacache = new SSKStore(this);
> - SaltedHashFreenetStore.construct(storeDir,
> "SSK-cache", sskDatacache,
random, maxCacheKeys,
> + SaltedHashFreenetStore sskCacheFS =
SaltedHashFreenetStore.construct(storeDir, "SSK-cache",
> + sskDatacache, random, maxCacheKeys,
> shutdownHook);
> +
> + File migrationFile = new File(storeDir,
> "migrated");
> + if (!migrationFile.exists()) {
> + chkDataFS.migrationFrom(//
> + new File(storeDir, "chk" +
> suffix + ".store"), //
> + new File(storeDir, "chk" +
> suffix + ".store.keys"));
> + chkCacheFS.migrationFrom(//
> + new File(storeDir, "chk" +
> suffix + ".cache"), //
> + new File(storeDir, "chk" +
> suffix + ".cache.keys"));
> +
> + pubkeyDataFS.migrationFrom(//
> + new File(storeDir, "pubkey" +
> suffix + ".store"), //
> + new File(storeDir, "pubkey" +
> suffix + ".store.keys"));
> + pubkeyCacheFS.migrationFrom(//
> + new File(storeDir, "pubkey" +
> suffix + ".cache"), //
> + new File(storeDir, "pubkey" +
> suffix + ".cache.keys"));
> +
> + sskDataFS.migrationFrom(//
> + new File(storeDir, "ssk" +
> suffix + ".store"), //
> + new File(storeDir, "ssk" +
> suffix + ".store.keys"));
> + sskCacheFS.migrationFrom(//
> + new File(storeDir, "ssk" +
> suffix + ".cache"), //
> + new File(storeDir, "ssk" +
> suffix + ".cache.keys"));
> + migrationFile.createNewFile();
> + }
> } catch (IOException e) {
> System.err.println("Could not open store: " +
> e);
> e.printStackTrace();
>
> Modified:
branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
> ===================================================================
> ---
branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
2008-05-26 09:34:05 UTC (rev 20096)
> +++
branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
2008-05-26 09:34:39 UTC (rev 20097)
> @@ -1376,4 +1376,47 @@
> return storeSize;
> }
>
> + // ------------- Migration
> + public void migrationFrom(File storeFile, File keyFile) {
> + try {
> + System.out.println("Migrating from " + storeFile);
> +
> + RandomAccessFile storeRAF = new
> RandomAccessFile(storeFile, "r");
> + RandomAccessFile keyRAF = keyFile.exists() ? new
RandomAccessFile(keyFile, "r") : null;
> +
> + byte[] header = new byte[headerBlockLength];
> + byte[] data = new byte[dataBlockLength];
> + byte[] key = new byte[fullKeyLength];
> +
> + long maxKey = storeRAF.length() / (headerBlockLength +
> dataBlockLength);
> + for (int l = 0; true; l++) {
> + if (l % 1024 == 0)
> + System.out.println(" key " + l + "/" +
> maxKey);
> +
> + boolean keyRead = false;
> + storeRAF.readFully(header);
> + storeRAF.readFully(data);
> + try {
> + if (keyRAF != null) {
> + keyRAF.readFully(key);
> + keyRead = true;
> + }
> + } catch (IOException e) {
> + }
> +
> + try {
> + StorableBlock b =
> callback.construct(data, header, null, keyRead ?
key : null);
> + put(b, b.getRoutingKey(),
> b.getFullKey(), data, header, true);
> + } catch (KeyVerifyException e) {
> + System.out.println("kve at block " + l);
> + } catch (KeyCollisionException e) {
> + System.out.println("kce at block " + l);
> + }
> + }
> + } catch (EOFException eof) {
> + // done
> + } catch (IOException e) {
> + e.printStackTrace();
> + }
> + }
> }
>
> _______________________________________________
> cvs mailing list
> cvs at freenetproject.org
> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs
>
>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 189 bytes
Desc: not available
URL:
<https://emu.freenetproject.org/pipermail/devl/attachments/20080528/d89ca3cd/attachment.pgp>
