On Thursday 15 May 2008 01:00, Daniel Cheng wrote: > On Thu, May 15, 2008 at 5:18 AM, Matthew Toseland > <[EMAIL PROTECTED]> wrote: > > On Friday 09 May 2008 18:02, [EMAIL PROTECTED] wrote: > >> Author: j16sdiz > >> Date: 2008-05-09 17:02:34 +0000 (Fri, 09 May 2008) > >> New Revision: 19881 > >> > >> Modified: > >> > > branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java > >> Log: > >> Store resize tweak > >> > >> > >> Modified: > > branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java > >> =================================================================== > >> --- > > branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java > > 2008-05-09 17:02:14 UTC (rev 19880) > >> +++ > > branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java > > 2008-05-09 17:02:34 UTC (rev 19881) > >> @@ -28,9 +28,6 @@ > >> import freenet.support.io.FileUtil; > >> import freenet.support.math.RunningAverage; > >> import freenet.support.math.SimpleRunningAverage; > >> -import java.util.ArrayList; > >> -import java.util.Iterator; > >> -import java.util.List; > >> > >> /** > >> * Index-less data store based on salted hash > >> @@ -602,7 +599,7 @@ > >> } > >> } > >> } > >> - > >> + > >> // ------------- Configuration > >> /** > >> * Configuration File > >> @@ -707,13 +704,9 @@ > >> } > >> > >> /** > >> - * Maximum memory to be used on resize > >> - */ > >> - private static final int RESIZE_MEMORY = 2 * 1024 * 1024; // 2MiB > >> - /** > >> * Phase 1 Rounds > >> */ > >> - private static final int RESIZE_PHASE1_ROUND = 12; > >> + private static final int RESIZE_PHASE1_ROUND = 8; > >> /** > >> * Maximum resize round > >> */ > >> @@ -788,9 +781,17 @@ > >> resolvedEntries = 0; > >> droppedEntries = 0; > >> > >> - List oldItems = null; > >> + File oldItemFile = null; > >> + RandomAccessFile oldItemsRAF = null; > >> + FileChannel oldItemsFC = null; > >> if (queueItem) { > >> - oldItems = new ArrayList(); > >> + try { > >> + oldItemFile = new File(baseDir, name + ".oldItems"); > >> + oldItemsRAF = new RandomAccessFile(oldItemFile, "rw"); > >> + oldItemsRAF.seek(oldItemsRAF.length()); > > > > If you are doing what I think you are doing, you should make sure where you > > seek to is a multiple of the record size. > > I am seeking to end of file, append to last record. > It should work as long as the file is not truncated. > (okay, it need better error handling)
If it crashes or whatever, it's important that you start writing on a record boundary. > > >> + oldItemsFC = oldItemsRAF.getChannel(); > >> + } catch (IOException e) { > >> + } > >> } > >> > >> long maxOffset = maxOldItemOffset; > >> @@ -852,16 +853,18 @@ > >> resolvedEntries++; > >> } else if (queueItem) { > >> // break tie by moveing old item to queue > >> - if (oldItems.size() * entryTotalLength < RESIZE_MEMORY) { > >> - oldItems.add(newOffsetEntry); > >> - if (newOffset > offset) { > >> - > >> oldEntries++; // newOffset wasn't counted count it > >> - } > >> + if (logDEBUG) > >> + Logger.debug(this, "Write item " > >> + > >> + HexUtil.bytesToHex(newOffsetEntry.digestedRoutingKey) > >> + > >> + " to old item file"); > >> + writeOldItem(oldItemsFC, newOffsetEntry); > >> + if (newOffset > offset) { > >> + oldEntries++; // newOffset wasn't counted count it > >> + } > >> > >> - writeEntry(entry); > >> - freeOffset(offset); > >> - resolvedEntries++; > >> - } > >> + writeEntry(entry); > >> + freeOffset(offset); > >> + resolvedEntries++; > >> } > >> } finally { > >> unlockEntry(newOffset); > >> @@ -875,25 +878,38 @@ > >> } > >> > >> if (queueItem) { > >> - putBackOldItems(oldItems); > >> + try { > >> + oldItemsRAF.seek(0); > >> + putBackOldItems(oldItemsFC); > >> + } catch (IOException e) { > >> + } finally { > >> + try { > >> + oldItemsRAF.close(); > >> + oldItemFile.delete(); > >> + } catch (IOException e2) { > >> + } > >> + } > >> } > >> } > >> > >> /** > >> * Put back oldItems with best effort > >> + * > >> + * @throws IOException > >> */ > >> - private void putBackOldItems(List oldItems) { > >> - Iterator it = oldItems.iterator(); > >> - while (it.hasNext()) { > >> - boolean done = false; > >> + private void putBackOldItems(FileChannel oldItems) throws IOException { > >> + while (true) { > >> + Entry entry = readOldItem(oldItems); > >> + if (entry == null) > >> + break; > >> > >> - Entry entry = (Entry) it.next(); > >> entry.setStoreSize(storeSize); > >> > >> long newOffset = entry.getOffset(); > >> > >> if (!lockEntry(newOffset)) // lock > >> continue; > >> + boolean done = false; > >> try { > >> if (isFree(newOffset)) { > >> if (logDEBUG) > >> @@ -908,7 +924,6 @@ > >> Logger.debug(this, "IOExcception on putBackOldItems", e); > >> } finally { > >> unlockEntry(newOffset); > >> - it.remove(); > >> > >> if (done) > >> resolvedEntries++; > >> @@ -916,9 +931,23 @@ > >> droppedEntries++; > >> } > >> } > >> - oldItems.clear(); > >> } > >> > >> + private void writeOldItem(FileChannel fc, Entry e) throws IOException { > >> + ByteBuffer bf = e.toByteBuffer(); > >> + do { > >> + fc.write(bf); > >> + } while (bf.hasRemaining()); > >> + } > >> + > >> + private Entry readOldItem(FileChannel fc) throws IOException { > >> + ByteBuffer bf = ByteBuffer.allocate((int) entryTotalLength); > >> + do { > >> + fc.read(bf); > >> + } while (bf.hasRemaining()); > >> + return new Entry(bf); > >> + } > >> + > >> /** > >> * Samples to take on key count estimation > >> */ > > > > Is there any limit on the size of the old items file? > > The current code have no limit on size. Unless you try shrink the > store to some insane small number, > it should have no more then a few hundred items. Ok. > > > Would it enable rekeying should we discover we need it? > > What kind of rekeying are you looking for? > if you are asking for new salt -- no, you need the route key for that > to do that immediatly.
pgpEiW8fUpDD5.pgp
Description: PGP signature
_______________________________________________ Devl mailing list Devl@freenetproject.org http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl