On Thu, May 15, 2008 at 5:18 AM, Matthew Toseland
<toad at amphibian.dyndns.org> wrote:
> On Friday 09 May 2008 18:02, j16sdiz at freenetproject.org wrote:
>> Author: j16sdiz
>> Date: 2008-05-09 17:02:34 +0000 (Fri, 09 May 2008)
>> New Revision: 19881
>>
>> Modified:
>>
> branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
>> Log:
>> Store resize tweak
>>
>>
>> Modified:
> branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
>> ===================================================================
>> ---
> branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
> 2008-05-09 17:02:14 UTC (rev 19880)
>> +++
> branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
> 2008-05-09 17:02:34 UTC (rev 19881)
>> @@ -28,9 +28,6 @@
>> import freenet.support.io.FileUtil;
>> import freenet.support.math.RunningAverage;
>> import freenet.support.math.SimpleRunningAverage;
>> -import java.util.ArrayList;
>> -import java.util.Iterator;
>> -import java.util.List;
>>
>> /**
>> * Index-less data store based on salted hash
>> @@ -602,7 +599,7 @@
>> }
>> }
>> }
>> -
>> +
>> // ------------- Configuration
>> /**
>> * Configuration File
>> @@ -707,13 +704,9 @@
>> }
>>
>> /**
>> - * Maximum memory to be used on resize
>> - */
>> - private static final int RESIZE_MEMORY = 2 * 1024 * 1024; //
>> 2MiB
>> - /**
>> * Phase 1 Rounds
>> */
>> - private static final int RESIZE_PHASE1_ROUND = 12;
>> + private static final int RESIZE_PHASE1_ROUND = 8;
>> /**
>> * Maximum resize round
>> */
>> @@ -788,9 +781,17 @@
>> resolvedEntries = 0;
>> droppedEntries = 0;
>>
>> - List oldItems = null;
>> + File oldItemFile = null;
>> + RandomAccessFile oldItemsRAF = null;
>> + FileChannel oldItemsFC = null;
>> if (queueItem) {
>> - oldItems = new ArrayList();
>> + try {
>> + oldItemFile = new File(baseDir, name +
>> ".oldItems");
>> + oldItemsRAF = new
>> RandomAccessFile(oldItemFile, "rw");
>> + oldItemsRAF.seek(oldItemsRAF.length());
>
> If you are doing what I think you are doing, you should make sure where you
> seek to is a multiple of the record size.
I am seeking to end of file, append to last record.
It should work as long as the file is not truncated.
(okay, it need better error handling)
>> + oldItemsFC = oldItemsRAF.getChannel();
>> + } catch (IOException e) {
>> + }
>> }
>>
>> long maxOffset = maxOldItemOffset;
>> @@ -852,16 +853,18 @@
>>
>> resolvedEntries++;
>> } else if (queueItem) {
>> // break tie
>> by moveing old item to queue
>> - if
>> (oldItems.size() * entryTotalLength < RESIZE_MEMORY) {
>> -
>> oldItems.add(newOffsetEntry);
>> - if
>> (newOffset > offset) {
>> -
>> oldEntries++; // newOffset wasn't counted count it
>> - }
>> + if (logDEBUG)
>> +
>> Logger.debug(this, "Write item "
>> +
>> + HexUtil.bytesToHex(newOffsetEntry.digestedRoutingKey)
>> +
>> + " to old item file");
>> +
>> writeOldItem(oldItemsFC, newOffsetEntry);
>> + if (newOffset
>> > offset) {
>> +
>> oldEntries++; // newOffset wasn't counted count it
>> + }
>>
>> -
>> writeEntry(entry);
>> -
>> freeOffset(offset);
>> -
>> resolvedEntries++;
>> - }
>> +
>> writeEntry(entry);
>> +
>> freeOffset(offset);
>> +
>> resolvedEntries++;
>> }
>> } finally {
>> unlockEntry(newOffset);
>> @@ -875,25 +878,38 @@
>> }
>>
>> if (queueItem) {
>> - putBackOldItems(oldItems);
>> + try {
>> + oldItemsRAF.seek(0);
>> + putBackOldItems(oldItemsFC);
>> + } catch (IOException e) {
>> + } finally {
>> + try {
>> + oldItemsRAF.close();
>> + oldItemFile.delete();
>> + } catch (IOException e2) {
>> + }
>> + }
>> }
>> }
>>
>> /**
>> * Put back oldItems with best effort
>> + *
>> + * @throws IOException
>> */
>> - private void putBackOldItems(List oldItems) {
>> - Iterator it = oldItems.iterator();
>> - while (it.hasNext()) {
>> - boolean done = false;
>> + private void putBackOldItems(FileChannel oldItems) throws
>> IOException {
>> + while (true) {
>> + Entry entry = readOldItem(oldItems);
>> + if (entry == null)
>> + break;
>>
>> - Entry entry = (Entry) it.next();
>> entry.setStoreSize(storeSize);
>>
>> long newOffset = entry.getOffset();
>>
>> if (!lockEntry(newOffset)) // lock
>> continue;
>> + boolean done = false;
>> try {
>> if (isFree(newOffset)) {
>> if (logDEBUG)
>> @@ -908,7 +924,6 @@
>> Logger.debug(this, "IOExcception on
>> putBackOldItems", e);
>> } finally {
>> unlockEntry(newOffset);
>> - it.remove();
>>
>> if (done)
>> resolvedEntries++;
>> @@ -916,9 +931,23 @@
>> droppedEntries++;
>> }
>> }
>> - oldItems.clear();
>> }
>>
>> + private void writeOldItem(FileChannel fc, Entry e) throws
>> IOException {
>> + ByteBuffer bf = e.toByteBuffer();
>> + do {
>> + fc.write(bf);
>> + } while (bf.hasRemaining());
>> + }
>> +
>> + private Entry readOldItem(FileChannel fc) throws IOException {
>> + ByteBuffer bf = ByteBuffer.allocate((int)
>> entryTotalLength);
>> + do {
>> + fc.read(bf);
>> + } while (bf.hasRemaining());
>> + return new Entry(bf);
>> + }
>> +
>> /**
>> * Samples to take on key count estimation
>> */
>
> Is there any limit on the size of the old items file?
The current code have no limit on size. Unless you try shrink the
store to some insane small number,
it should have no more then a few hundred items.
> Would it enable rekeying should we discover we need it?
What kind of rekeying are you looking for?
if you are asking for new salt -- no, you need the route key for that
to do that immediatly.
