On Friday 09 May 2008 18:02, j16sdiz at freenetproject.org wrote:
> Author: j16sdiz
> Date: 2008-05-09 17:02:34 +0000 (Fri, 09 May 2008)
> New Revision: 19881
>
> Modified:
>
branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
> Log:
> Store resize tweak
>
>
> Modified:
branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
> ===================================================================
> ---
branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
2008-05-09 17:02:14 UTC (rev 19880)
> +++
branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
2008-05-09 17:02:34 UTC (rev 19881)
> @@ -28,9 +28,6 @@
> import freenet.support.io.FileUtil;
> import freenet.support.math.RunningAverage;
> import freenet.support.math.SimpleRunningAverage;
> -import java.util.ArrayList;
> -import java.util.Iterator;
> -import java.util.List;
>
> /**
> * Index-less data store based on salted hash
> @@ -602,7 +599,7 @@
> }
> }
> }
> -
> +
> // ------------- Configuration
> /**
> * Configuration File
> @@ -707,13 +704,9 @@
> }
>
> /**
> - * Maximum memory to be used on resize
> - */
> - private static final int RESIZE_MEMORY = 2 * 1024 * 1024; //
> 2MiB
> - /**
> * Phase 1 Rounds
> */
> - private static final int RESIZE_PHASE1_ROUND = 12;
> + private static final int RESIZE_PHASE1_ROUND = 8;
> /**
> * Maximum resize round
> */
> @@ -788,9 +781,17 @@
> resolvedEntries = 0;
> droppedEntries = 0;
>
> - List oldItems = null;
> + File oldItemFile = null;
> + RandomAccessFile oldItemsRAF = null;
> + FileChannel oldItemsFC = null;
> if (queueItem) {
> - oldItems = new ArrayList();
> + try {
> + oldItemFile = new File(baseDir, name +
> ".oldItems");
> + oldItemsRAF = new
> RandomAccessFile(oldItemFile, "rw");
> + oldItemsRAF.seek(oldItemsRAF.length());
If you are doing what I think you are doing, you should make sure where you
seek to is a multiple of the record size.
> + oldItemsFC = oldItemsRAF.getChannel();
> + } catch (IOException e) {
> + }
> }
>
> long maxOffset = maxOldItemOffset;
> @@ -852,16 +853,18 @@
>
> resolvedEntries++;
> } else if (queueItem) {
> // break tie by
> moveing old item to queue
> - if
> (oldItems.size() * entryTotalLength < RESIZE_MEMORY) {
> -
> oldItems.add(newOffsetEntry);
> - if
> (newOffset > offset) {
> -
> oldEntries++; // newOffset wasn't counted count it
> - }
> + if (logDEBUG)
> +
> Logger.debug(this, "Write item "
> +
> + HexUtil.bytesToHex(newOffsetEntry.digestedRoutingKey)
> +
> + " to old item file");
> +
> writeOldItem(oldItemsFC, newOffsetEntry);
> + if (newOffset >
> offset) {
> +
> oldEntries++; // newOffset wasn't counted count it
> + }
>
> -
> writeEntry(entry);
> -
> freeOffset(offset);
> -
> resolvedEntries++;
> - }
> +
> writeEntry(entry);
> +
> freeOffset(offset);
> +
> resolvedEntries++;
> }
> } finally {
> unlockEntry(newOffset);
> @@ -875,25 +878,38 @@
> }
>
> if (queueItem) {
> - putBackOldItems(oldItems);
> + try {
> + oldItemsRAF.seek(0);
> + putBackOldItems(oldItemsFC);
> + } catch (IOException e) {
> + } finally {
> + try {
> + oldItemsRAF.close();
> + oldItemFile.delete();
> + } catch (IOException e2) {
> + }
> + }
> }
> }
>
> /**
> * Put back oldItems with best effort
> + *
> + * @throws IOException
> */
> - private void putBackOldItems(List oldItems) {
> - Iterator it = oldItems.iterator();
> - while (it.hasNext()) {
> - boolean done = false;
> + private void putBackOldItems(FileChannel oldItems) throws
> IOException {
> + while (true) {
> + Entry entry = readOldItem(oldItems);
> + if (entry == null)
> + break;
>
> - Entry entry = (Entry) it.next();
> entry.setStoreSize(storeSize);
>
> long newOffset = entry.getOffset();
>
> if (!lockEntry(newOffset)) // lock
> continue;
> + boolean done = false;
> try {
> if (isFree(newOffset)) {
> if (logDEBUG)
> @@ -908,7 +924,6 @@
> Logger.debug(this, "IOExcception on
> putBackOldItems", e);
> } finally {
> unlockEntry(newOffset);
> - it.remove();
>
> if (done)
> resolvedEntries++;
> @@ -916,9 +931,23 @@
> droppedEntries++;
> }
> }
> - oldItems.clear();
> }
>
> + private void writeOldItem(FileChannel fc, Entry e) throws
> IOException {
> + ByteBuffer bf = e.toByteBuffer();
> + do {
> + fc.write(bf);
> + } while (bf.hasRemaining());
> + }
> +
> + private Entry readOldItem(FileChannel fc) throws IOException {
> + ByteBuffer bf = ByteBuffer.allocate((int)
> entryTotalLength);
> + do {
> + fc.read(bf);
> + } while (bf.hasRemaining());
> + return new Entry(bf);
> + }
> +
> /**
> * Samples to take on key count estimation
> */
Is there any limit on the size of the old items file? Would it enable rekeying
should we discover we need it?
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 189 bytes
Desc: not available
URL:
<https://emu.freenetproject.org/pipermail/devl/attachments/20080514/f8252cf8/attachment.pgp>
