On Thursday 15 May 2008 01:00, Daniel Cheng wrote:
> On Thu, May 15, 2008 at 5:18 AM, Matthew Toseland
> <toad at amphibian.dyndns.org> wrote:
> > On Friday 09 May 2008 18:02, j16sdiz at freenetproject.org wrote:
> >> Author: j16sdiz
> >> Date: 2008-05-09 17:02:34 +0000 (Fri, 09 May 2008)
> >> New Revision: 19881
> >>
> >> Modified:
> >>
> >
branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
> >> Log:
> >> Store resize tweak
> >>
> >>
> >> Modified:
> >
branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
> >> ===================================================================
> >> ---
> >
branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
> > 2008-05-09 17:02:14 UTC (rev 19880)
> >> +++
> >
branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
> > 2008-05-09 17:02:34 UTC (rev 19881)
> >> @@ -28,9 +28,6 @@
> >> import freenet.support.io.FileUtil;
> >> import freenet.support.math.RunningAverage;
> >> import freenet.support.math.SimpleRunningAverage;
> >> -import java.util.ArrayList;
> >> -import java.util.Iterator;
> >> -import java.util.List;
> >>
> >> /**
> >> * Index-less data store based on salted hash
> >> @@ -602,7 +599,7 @@
> >> }
> >> }
> >> }
> >> -
> >> +
> >> // ------------- Configuration
> >> /**
> >> * Configuration File
> >> @@ -707,13 +704,9 @@
> >> }
> >>
> >> /**
> >> - * Maximum memory to be used on resize
> >> - */
> >> - private static final int RESIZE_MEMORY = 2 * 1024 *
1024; // 2MiB
> >> - /**
> >> * Phase 1 Rounds
> >> */
> >> - private static final int RESIZE_PHASE1_ROUND = 12;
> >> + private static final int RESIZE_PHASE1_ROUND = 8;
> >> /**
> >> * Maximum resize round
> >> */
> >> @@ -788,9 +781,17 @@
> >> resolvedEntries = 0;
> >> droppedEntries = 0;
> >>
> >> - List oldItems = null;
> >> + File oldItemFile = null;
> >> + RandomAccessFile oldItemsRAF = null;
> >> + FileChannel oldItemsFC = null;
> >> if (queueItem) {
> >> - oldItems = new ArrayList();
> >> + try {
> >> + oldItemFile = new File(baseDir,
name + ".oldItems");
> >> + oldItemsRAF = new
RandomAccessFile(oldItemFile, "rw");
> >> +
oldItemsRAF.seek(oldItemsRAF.length());
> >
> > If you are doing what I think you are doing, you should make sure where
you
> > seek to is a multiple of the record size.
>
> I am seeking to end of file, append to last record.
> It should work as long as the file is not truncated.
> (okay, it need better error handling)
If it crashes or whatever, it's important that you start writing on a record
boundary.
>
> >> + oldItemsFC =
oldItemsRAF.getChannel();
> >> + } catch (IOException e) {
> >> + }
> >> }
> >>
> >> long maxOffset = maxOldItemOffset;
> >> @@ -852,16 +853,18 @@
> >>
resolvedEntries++;
> >> } else if
(queueItem) {
> >> // break
tie by moveing old item to queue
> >> - if
(oldItems.size() * entryTotalLength < RESIZE_MEMORY) {
> >> -
oldItems.add(newOffsetEntry);
> >> - if
(newOffset > offset) {
> >> -
> >>
oldEntries++; // newOffset wasn't counted count it
> >> - }
> >> + if
(logDEBUG)
> >> +
Logger.debug(this, "Write item "
> >> +
> >>
+ HexUtil.bytesToHex(newOffsetEntry.digestedRoutingKey)
> >> +
> >>
+ " to old item file");
> >> +
writeOldItem(oldItemsFC, newOffsetEntry);
> >> + if
(newOffset > offset) {
> >> +
oldEntries++; // newOffset wasn't counted count it
> >> + }
> >>
> >> -
writeEntry(entry);
> >> -
freeOffset(offset);
> >> -
resolvedEntries++;
> >> - }
> >> +
writeEntry(entry);
> >> +
freeOffset(offset);
> >> +
resolvedEntries++;
> >> }
> >> } finally {
> >>
unlockEntry(newOffset);
> >> @@ -875,25 +878,38 @@
> >> }
> >>
> >> if (queueItem) {
> >> - putBackOldItems(oldItems);
> >> + try {
> >> + oldItemsRAF.seek(0);
> >> + putBackOldItems(oldItemsFC);
> >> + } catch (IOException e) {
> >> + } finally {
> >> + try {
> >> + oldItemsRAF.close();
> >> + oldItemFile.delete();
> >> + } catch (IOException e2) {
> >> + }
> >> + }
> >> }
> >> }
> >>
> >> /**
> >> * Put back oldItems with best effort
> >> + *
> >> + * @throws IOException
> >> */
> >> - private void putBackOldItems(List oldItems) {
> >> - Iterator it = oldItems.iterator();
> >> - while (it.hasNext()) {
> >> - boolean done = false;
> >> + private void putBackOldItems(FileChannel oldItems) throws
IOException {
> >> + while (true) {
> >> + Entry entry = readOldItem(oldItems);
> >> + if (entry == null)
> >> + break;
> >>
> >> - Entry entry = (Entry) it.next();
> >> entry.setStoreSize(storeSize);
> >>
> >> long newOffset = entry.getOffset();
> >>
> >> if (!lockEntry(newOffset)) // lock
> >> continue;
> >> + boolean done = false;
> >> try {
> >> if (isFree(newOffset)) {
> >> if (logDEBUG)
> >> @@ -908,7 +924,6 @@
> >> Logger.debug(this, "IOExcception on
putBackOldItems", e);
> >> } finally {
> >> unlockEntry(newOffset);
> >> - it.remove();
> >>
> >> if (done)
> >> resolvedEntries++;
> >> @@ -916,9 +931,23 @@
> >> droppedEntries++;
> >> }
> >> }
> >> - oldItems.clear();
> >> }
> >>
> >> + private void writeOldItem(FileChannel fc, Entry e) throws
IOException {
> >> + ByteBuffer bf = e.toByteBuffer();
> >> + do {
> >> + fc.write(bf);
> >> + } while (bf.hasRemaining());
> >> + }
> >> +
> >> + private Entry readOldItem(FileChannel fc) throws
IOException {
> >> + ByteBuffer bf = ByteBuffer.allocate((int)
entryTotalLength);
> >> + do {
> >> + fc.read(bf);
> >> + } while (bf.hasRemaining());
> >> + return new Entry(bf);
> >> + }
> >> +
> >> /**
> >> * Samples to take on key count estimation
> >> */
> >
> > Is there any limit on the size of the old items file?
>
> The current code have no limit on size. Unless you try shrink the
> store to some insane small number,
> it should have no more then a few hundred items.
Ok.
>
> > Would it enable rekeying should we discover we need it?
>
> What kind of rekeying are you looking for?
> if you are asking for new salt -- no, you need the route key for that
> to do that immediatly.
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 189 bytes
Desc: not available
URL:
<https://emu.freenetproject.org/pipermail/devl/attachments/20080515/843deb47/attachment.pgp>
