On Friday 09 May 2008 18:02, [EMAIL PROTECTED] wrote: > Author: j16sdiz > Date: 2008-05-09 17:02:34 +0000 (Fri, 09 May 2008) > New Revision: 19881 > > Modified: > branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java > Log: > Store resize tweak > > > Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java > =================================================================== > --- branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-05-09 17:02:14 UTC (rev 19880) > +++ branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-05-09 17:02:34 UTC (rev 19881) > @@ -28,9 +28,6 @@ > import freenet.support.io.FileUtil; > import freenet.support.math.RunningAverage; > import freenet.support.math.SimpleRunningAverage; > -import java.util.ArrayList; > -import java.util.Iterator; > -import java.util.List; > > /** > * Index-less data store based on salted hash > @@ -602,7 +599,7 @@ > } > } > } > - > + > // ------------- Configuration > /** > * Configuration File > @@ -707,13 +704,9 @@ > } > > /** > - * Maximum memory to be used on resize > - */ > - private static final int RESIZE_MEMORY = 2 * 1024 * 1024; // > 2MiB > - /** > * Phase 1 Rounds > */ > - private static final int RESIZE_PHASE1_ROUND = 12; > + private static final int RESIZE_PHASE1_ROUND = 8; > /** > * Maximum resize round > */ > @@ -788,9 +781,17 @@ > resolvedEntries = 0; > droppedEntries = 0; > > - List oldItems = null; > + File oldItemFile = null; > + RandomAccessFile oldItemsRAF = null; > + FileChannel oldItemsFC = null; > if (queueItem) { > - oldItems = new ArrayList(); > + try { > + oldItemFile = new File(baseDir, name + > ".oldItems"); > + oldItemsRAF = new > RandomAccessFile(oldItemFile, "rw"); > + oldItemsRAF.seek(oldItemsRAF.length());
If you are doing what I think you are doing, you should make sure where you seek to is a multiple of the record size. > + oldItemsFC = oldItemsRAF.getChannel(); > + } catch (IOException e) { > + } > } > > long maxOffset = maxOldItemOffset; > @@ -852,16 +853,18 @@ > > resolvedEntries++; > } else if (queueItem) { > // break tie by > moveing old item to queue > - if > (oldItems.size() * entryTotalLength < RESIZE_MEMORY) { > - > oldItems.add(newOffsetEntry); > - if > (newOffset > offset) { > - > oldEntries++; // newOffset wasn't counted count it > - } > + if (logDEBUG) > + > Logger.debug(this, "Write item " > + > + HexUtil.bytesToHex(newOffsetEntry.digestedRoutingKey) > + > + " to old item file"); > + > writeOldItem(oldItemsFC, newOffsetEntry); > + if (newOffset > > offset) { > + > oldEntries++; // newOffset wasn't counted count it > + } > > - > writeEntry(entry); > - > freeOffset(offset); > - > resolvedEntries++; > - } > + > writeEntry(entry); > + > freeOffset(offset); > + > resolvedEntries++; > } > } finally { > unlockEntry(newOffset); > @@ -875,25 +878,38 @@ > } > > if (queueItem) { > - putBackOldItems(oldItems); > + try { > + oldItemsRAF.seek(0); > + putBackOldItems(oldItemsFC); > + } catch (IOException e) { > + } finally { > + try { > + oldItemsRAF.close(); > + oldItemFile.delete(); > + } catch (IOException e2) { > + } > + } > } > } > > /** > * Put back oldItems with best effort > + * > + * @throws IOException > */ > - private void putBackOldItems(List oldItems) { > - Iterator it = oldItems.iterator(); > - while (it.hasNext()) { > - boolean done = false; > + private void putBackOldItems(FileChannel oldItems) throws > IOException { > + while (true) { > + Entry entry = readOldItem(oldItems); > + if (entry == null) > + break; > > - Entry entry = (Entry) it.next(); > entry.setStoreSize(storeSize); > > long newOffset = entry.getOffset(); > > if (!lockEntry(newOffset)) // lock > continue; > + boolean done = false; > try { > if (isFree(newOffset)) { > if (logDEBUG) > @@ -908,7 +924,6 @@ > Logger.debug(this, "IOExcception on > putBackOldItems", e); > } finally { > unlockEntry(newOffset); > - it.remove(); > > if (done) > resolvedEntries++; > @@ -916,9 +931,23 @@ > droppedEntries++; > } > } > - oldItems.clear(); > } > > + private void writeOldItem(FileChannel fc, Entry e) throws > IOException { > + ByteBuffer bf = e.toByteBuffer(); > + do { > + fc.write(bf); > + } while (bf.hasRemaining()); > + } > + > + private Entry readOldItem(FileChannel fc) throws IOException { > + ByteBuffer bf = ByteBuffer.allocate((int) > entryTotalLength); > + do { > + fc.read(bf); > + } while (bf.hasRemaining()); > + return new Entry(bf); > + } > + > /** > * Samples to take on key count estimation > */ Is there any limit on the size of the old items file? Would it enable rekeying should we discover we need it?
pgpqirXHWSXO4.pgp
Description: PGP signature
_______________________________________________ Devl mailing list Devl@freenetproject.org http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl