[freenet-dev] [freenet-cvs] r19857 - branches/saltedhashstore/freenet/src/freenet/store
gt; +
> freeOffset(offset);
> +
> resolvedEntries++;
> + } else if (queueItem) {
> + // break tie by
> moveing old item to queue
> + if
> (oldItems.size() * entryTotalLength < RESIZE_MEMORY) {
> +
> oldItems.add(newOffsetEntry);
> + if
> (newOffset > offset) {
> +
> oldEntries++; // newOffset wasn't counted count it
> + }
> +
> +
> writeEntry(entry);
> +
> freeOffset(offset);
> +
> resolvedEntries++;
Where are we writing it to here? It looks like we write it to the old
location, and then delete it?
> + }
> + }
> + } finally {
> + unlockEntry(newOffset);
> + }
> + }
> + } catch (IOException e) {
> + Logger.debug(this, "IOExcception on
> moveOldEntries0", e);
> + } finally {
> + unlockEntry(offset);
> + }
> + }
> +
> + if (queueItem) {
> + putBackOldItems(oldItems);
> + }
> + }
-- next part --
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 189 bytes
Desc: not available
URL:
<https://emu.freenetproject.org/pipermail/devl/attachments/20080509/78235e59/attachment.pgp>
[freenet-dev] [freenet-cvs] r19789 - branches/saltedhashstore/freenet/src/freenet/store
Buffer toByteBuffer() {
> ByteBuffer out = ByteBuffer.allocate((int)
> entryTotalLength);
> encrypt();
> - out.put(routingKey);
> + out.put(encryptedRoutingKey);
> out.put(dataEncryptIV);
>
> out.putLong(flag);
> out.putLong(storeSize);
>
> + if ((flag & ENTRY_FLAG_PLAINKEY) != 0) {
> + out.put(plainRoutingKey);
> + }
> +
> // reserved bytes
> out.position((int) ENTRY_HEADER_LENGTH);
>
> @@ -316,9 +336,9 @@
>
> public long getOffset() {
> if (isEncrypted)
> - return getOffsetFromDigestedKey(routingKey,
> storeSize);
> + return
> getOffsetFromDigestedKey(encryptedRoutingKey, storeSize);
> else
> - return getOffsetFromPlainKey(routingKey,
> storeSize);
> + return getOffsetFromPlainKey(plainRoutingKey,
> storeSize);
> }
>
> /**
> @@ -331,21 +351,22 @@
> private boolean decrypt(byte[] routingKey) {
> if (!isEncrypted) {
> // Already decrypted
> - if (Arrays.equals(this.routingKey, routingKey))
> + if (Arrays.equals(this.plainRoutingKey,
> routingKey))
> return true;
> else
> return false;
> }
>
> // Does the digested routing key match?
> - if (!Arrays.equals(this.routingKey,
> getDigestedRoutingKey(routingKey)))
> + if (!Arrays.equals(this.encryptedRoutingKey,
getDigestedRoutingKey(routingKey)))
> return false;
>
> - PCFBMode cipher = makeCipher(routingKey);
> + this.plainRoutingKey = routingKey;
> +
> + PCFBMode cipher = makeCipher(plainRoutingKey);
> header = cipher.blockDecipher(header, 0, header.length);
> data = cipher.blockDecipher(data, 0, data.length);
>
> - this.routingKey = routingKey;
> isEncrypted = false;
>
> return true;
> @@ -361,11 +382,11 @@
> dataEncryptIV = new byte[16];
> random.nextBytes(dataEncryptIV);
>
> - PCFBMode cipher = makeCipher(routingKey);
> + PCFBMode cipher = makeCipher(plainRoutingKey);
> header = cipher.blockEncipher(header, 0, header.length);
> data = cipher.blockEncipher(data, 0, data.length);
>
> - routingKey = getDigestedRoutingKey(routingKey);
> + encryptedRoutingKey =
> getDigestedRoutingKey(plainRoutingKey);
> isEncrypted = true;
> }
>
>
> ___
> cvs mailing list
> cvs at freenetproject.org
> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs
>
>
-- next part --
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 189 bytes
Desc: not available
URL:
<https://emu.freenetproject.org/pipermail/devl/attachments/20080509/a618e932/attachment.pgp>
[freenet-dev] [freenet-cvs] r19733 - branches/saltedhashstore/freenet/src/freenet/store
On Sunday 04 May 2008 14:11, you wrote:
> Author: j16sdiz
> Date: 2008-05-04 13:11:55 + (Sun, 04 May 2008)
> New Revision: 19733
>
> Added:
>
branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
> Log:
> SaltedHashFreenetStore basic function (without locking/resizing)
>
>
> Added:
branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
> ===
> ---
branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
(rev 0)
> +++
branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
2008-05-04 13:11:55 UTC (rev 19733)
> +
> + public StorableBlock getStorableBlock(byte[] routingKey, byte[]
> fullKey)
throws KeyVerifyException {
> + if (!decrypt(routingKey))
> + return null;
> +
> + StorableBlock block = callback.construct(data, header,
> routingKey,
fullKey);
> + byte[] blockRoutingKey = block.getRoutingKey();
> +
> + if (!Arrays.equals(blockRoutingKey, routingKey)) {
> + // either the data is corrupted or we have
> found a SHA-1 collision
> + // can't recover, as decrypt() depends on a
> correct route key
> + return null;
> + }
> +
> + return block;
> + }
If we can't be sure that the Entry isn't re-encrypted, we should copy the
data/headers here. If we can, we should document it.
> +
> + /**
> + * Check if a block is free
> + *
> + * @param offset
> + * @throws IOException
> + */
> + private boolean isFree(long offset) throws IOException {
> + int split = (int) (offset % FILE_SPLIT);
> + long rawOffset = (offset / FILE_SPLIT) * entryTotalLength;
> +
> + ByteBuffer bf = ByteBuffer.allocate((int) ENTRY_HEADER_LENGTH);
> +
> + do {
> + int status = storeFC[split].write(bf, rawOffset +
> bf.position());
> + if (status == -1)
> + throw new EOFException();
> + } while (bf.hasRemaining());
> +
> + return (bf.getLong(0x30) & ENTRY_FLAG_OCCUPIED) == 0;
> + }
Why read the whole entry just to check one byte (or one long)?
-- next part --
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 189 bytes
Desc: not available
URL:
<https://emu.freenetproject.org/pipermail/devl/attachments/20080509/a2343b41/attachment.pgp>
Re: [freenet-dev] [freenet-cvs] r19857 - branches/saltedhashstore/freenet/src/freenet/store
On Sat, May 10, 2008 at 6:39 AM, Matthew Toseland
<[EMAIL PROTECTED]> wrote:
> On Friday 09 May 2008 03:09, [EMAIL PROTECTED] wrote:
>>
>> /**
>> - * Sample to take at a time
>> + * Scan all entries and try to move them
>>*/
>> - private static final double SAMPLE_RATE = 0.05; // 5%
>> + private void moveOldEntry0(boolean queueItem) {
>> + newEntries = 0;
>> + oldEntries = 0;
>> + freeEntries = 0;
>> + resolvedEntries = 0;
>> + droppedEntries = 0;
>>
>> + List oldItems = null;
>> + if (queueItem) {
>> + oldItems = new ArrayList();
>> + }
>> +
>> + long maxOffset = maxOldItemOffset;
>> + maxOldItemOffset = 0;
>> + for (long offset = 0; offset <= maxOffset; offset++) {
>> + if (logDEBUG && offset % 1024 == 0) {
>> + Logger.debug(this, "Resize progress:
>> newEntries=" + newEntries + ",
> oldEntries=" + oldEntries
>> + + ", freeEntries=" +
>> freeEntries + ", resolvedEntries=" +
> resolvedEntries
>> + + ", droppedEntries=" +
>> droppedEntries);
>> + }
>> +
>> + if (shutdown)
>> + return;
>> +
>> + if (!lockEntry(offset)) //lock
>> + continue;
>> + try {
>> + Entry entry = readEntry(offset, null);
[]
>> + } else { // if (entry.getStoreSize()
>> == prevStoreSize)
>> + // old store size entries, try
>> to move them
>> + oldEntries++;
>> + maxOldItemOffset = offset;
>> +
>> + entry.setStoreSize(storeSize);
>> + long newOffset =
>> entry.getOffset();
[]
>> + } else if (queueItem) {
>> + // break tie
>> by moveing old item to queue
>> + if
>> (oldItems.size() * entryTotalLength < RESIZE_MEMORY) {
>> +
>> oldItems.add(newOffsetEntry);
>> + if
>> (newOffset > offset) {
>> +
>> oldEntries++; // newOffset wasn't counted count it
>> + }
>> +
>> +
>> writeEntry(entry);
>> +
>> freeOffset(offset);
>> +
>> resolvedEntries++;
>
> Where are we writing it to here? It looks like we write it to the old
> location, and then delete it?
>
"entry" need to migrate to newOffset, but newOffset is already
occupied by "newOffsetEntry", which also need migration. If
"queueItem" is true, we put "newOffsetEntry" to a queue and write
"entry" to "newOffset"
___
Devl mailing list
Devl@freenetproject.org
http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl
Re: [freenet-dev] [freenet-cvs] r19789 - branches/saltedhashstore/freenet/src/freenet/store
On Sat, May 10, 2008 at 6:24 AM, Matthew Toseland <[EMAIL PROTECTED]> wrote: > On Tuesday 06 May 2008 08:39, [EMAIL PROTECTED] wrote: >> Author: j16sdiz >> Date: 2008-05-06 07:39:43 + (Tue, 06 May 2008) >> New Revision: 19789 >> >> Modified: >> > branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java >> Log: >> option for saving plain key (for migration) > > For datastore histograms? It ought to be possible to make migration possible > without knowing the full key ... but it would require that the salt isn't > used in crypto, or is easily reversed out... hmmm. >> This code save all necessary keys to allow migrate _back_ to BDBFS. I don't want to lost any data in this early testing stage. ___ Devl mailing list Devl@freenetproject.org http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl
[freenet-dev] Post-0.7.0 priorities
friendly config sub-page for it. Average bandwidth limits, maybe actually > telling the node a monthly quota, would also be useful for many users. More > accurate bandwidth limiting (limiting all packets not just transfers) would > also be a big help for users on slower upstream connections. > Timescale: Unknown. > Priority: Medium. > > Allocate temporary files out of a small number of blob files: Further > reduction in memory usage, some other benefits, cuts the number of fd's we > use (thus allowing more FEC threads on mutli-core systems), easy. > Timescale: 3 days > Priority: Low. > Token passing load limiting. Just how important is this? It's likely to be a fair amount of work - we need to figure out a way to simulate it first off! It would give us more control over load management, hopefully enable us to use more bandwidth on faster nodes, might improve routing slightly, and have some security benefits... otoh the current code is *mostly* working adequately... My inclination is to postpone this for now, unless somebody wants to make a serious effort at simulating it? -- next part -- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available URL: <https://emu.freenetproject.org/pipermail/devl/attachments/20080509/10802c67/attachment.pgp>
[freenet-dev] What to do with the 0.5 related stuffs?
* Ringo Kamens <2600denver at gmail.com> [2008-05-08 23:35:20]: > I think we should keep .5 stuff around, since a lot of things link to > it We can provide permanent redirects to new content, that's not a problem. > but there's no reason to advertise it. Also, I get the feeling that > lots of freenet users don't trust .7 yet and that the .5 community > will continue to flourish for quite some time. Where are those users? that's the big question... They don't use our infrastructure anymore that's for sure. The number of hits on the .5 pages is ridiculous; We don't get much downloads either... The only service some are still using is the seednode-generation one. -- next part -- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 189 bytes Desc: Digital signature URL: <https://emu.freenetproject.org/pipermail/devl/attachments/20080509/77b2c2ed/attachment.pgp>
[freenet-dev] Post-0.7.0 priorities
On Friday 09 May 2008 07:27, Victor Denisov wrote: > | Automatic bandwidth calibration. Other p2p apps have this, we should > have it. > > Good idea. Also, we should definitely look into better utilizing > available bandwidth. Freenet's the only p2p app which consistently > underutilizes my upload limit (~ 2 Mbit/s out of 8 Mbit/s of link > capacity). I understand that we don't want to create supernodes, but > come on, 2 Mbit/s is *nothing* these days. IMHO automatic bandwidth calibration will help a lot with this. Beyond that we're looking at token passing, which may be too big for 0.7.1. > > | Client layer changes: I propose to move the entire client layer onto > disk. We > > I say this deserves to be moved to High or Very High priority. The main > problem is not memory usage as is (most people have 1 Gb+ of RAM now), > but rather inability of Java to properly grow and shrink its memory > usage on the "as needed" basis. Not a single native Windows application > behaves this way, and I doubt many users are prepared to understand and > adjust memory limits manually. Heck, even I, being a Java developer, > spent several days trying to understand what heap limit to set for > Freenet so that it won't run out of memory (and that it uses ~ 2x memory > on 64-bit JVM doesn't help any). Also, perhaps we can detect OOM errors > and offer the user to increase the memory limit next time he tries to > run Freenet? Agreed, memory usage is a usability issue: the user shouldn't have to care about it. > > | Auto-update for plugins: We should have had this ages ago. Several > people have > > Shouldn't we consider auto-updating bundled applications as well? Or > perhaps providing an auto-update API for use by third-party apps? Just a > thought. Maybe, that would be harder though. I would be happy to discuss it with their authors. > > Hope the above was helpful, > > Regards, > Victor Denisov. -- next part -- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available URL: <https://emu.freenetproject.org/pipermail/devl/attachments/20080509/38d17855/attachment.pgp>
[freenet-dev] Content Management System
* Michael T?nzer [2008-05-08 22:47:28]: > Ian Clarke schrieb: > > You could argue that all of those things have it easy, because most > > people understand what those things do, they don't need an elaborate > > explanation. But look at Gnome's website: > > > > http://www.gnome.org/ > > > > It is clean, simple, yet if you need to you can quickly dig down to a > > vast wealth of information. > > > > Gnome uses Plone as it's CMS. This may be not the best choice for us > though, as Plone is Phyton based, but that's something nextgens might > know better than me. > I don't have any problem with using python; Trac is in python too iirc... But I don't think that Plone is what we need, it's a framework more than a CMS. > > Either way, since we are Java hackers for the most part, not web > > designers, I strongly recommend that we borrow as much as we can from > > elsewhere, even so far as using free or creative commons HTML and CSS > > verbatim, perhaps with only a few minor changes. > > What about java-based CMSes then ? :p -- next part -- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 189 bytes Desc: Digital signature URL: <https://emu.freenetproject.org/pipermail/devl/attachments/20080509/157a20f2/attachment.pgp>
Re: [freenet-dev] [freenet-cvs] r19857 - branches/saltedhashstore/freenet/src/freenet/store
On Friday 09 May 2008 03:09, [EMAIL PROTECTED] wrote:
>
> /**
> - * Sample to take at a time
> + * Scan all entries and try to move them
>*/
> - private static final double SAMPLE_RATE = 0.05; // 5%
> + private void moveOldEntry0(boolean queueItem) {
> + newEntries = 0;
> + oldEntries = 0;
> + freeEntries = 0;
> + resolvedEntries = 0;
> + droppedEntries = 0;
>
> + List oldItems = null;
> + if (queueItem) {
> + oldItems = new ArrayList();
> + }
> +
> + long maxOffset = maxOldItemOffset;
> + maxOldItemOffset = 0;
> + for (long offset = 0; offset <= maxOffset; offset++) {
> + if (logDEBUG && offset % 1024 == 0) {
> + Logger.debug(this, "Resize progress:
> newEntries=" + newEntries + ",
oldEntries=" + oldEntries
> + + ", freeEntries=" +
> freeEntries + ", resolvedEntries=" +
resolvedEntries
> + + ", droppedEntries=" +
> droppedEntries);
> + }
> +
> + if (shutdown)
> + return;
> +
> + if (!lockEntry(offset)) //lock
> + continue;
> + try {
> + Entry entry = readEntry(offset, null);
> +
> + if (entry.isFree()) {
> + // free block
> + freeEntries++;
> + } else if (entry.getStoreSize() ==
> storeSize) {
> + // new store size entries
> + maxOldItemOffset = offset;
> + newEntries++;
> + } else { // if (entry.getStoreSize() ==
> prevStoreSize)
> + // old store size entries, try
> to move them
> + oldEntries++;
> + maxOldItemOffset = offset;
> +
> + entry.setStoreSize(storeSize);
> + long newOffset =
> entry.getOffset();
> +
> + if (newOffset == offset) { //
> lucky!
> + writeEntry(entry); //
> write back entry storeSize
> + resolvedEntries++;
> + continue;
> + }
> +
> + if (!lockEntry(newOffset)) //
> lock
> + continue;
> + try {
> + // see what's in the
> new offset
> + Entry newOffsetEntry =
> readEntry(newOffset, null);
> +
> + if
> (newOffsetEntry.isFree()) {
> + // the new
> offset is fr..
> +
> writeEntry(entry);
> +
> freeOffset(offset);
> +
> resolvedEntries++;
> + } else if
> (newOffsetEntry.getStoreSize() == storeSize) {
> + // new offset
> already have a new entry, free old entry
> +
> freeOffset(offset);
> +
> droppedEntries++;
> + } else if
> (Arrays.equals(entry.digestedRoutingKey,
newOffsetEntry.digestedRoutingKey)) {
> + // same
> digested routing key, free the old entry
> +
> freeOffset(offset);
> +
> resolvedEntries++;
> + } else if (queueItem) {
> + // break tie by
> moveing old item to queue
> +
Re: [freenet-dev] [freenet-cvs] r19789 - branches/saltedhashstore/freenet/src/freenet/store
On Tuesday 06 May 2008 08:39, [EMAIL PROTECTED] wrote:
> Author: j16sdiz
> Date: 2008-05-06 07:39:43 + (Tue, 06 May 2008)
> New Revision: 19789
>
> Modified:
>
branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
> Log:
> option for saving plain key (for migration)
For datastore histograms? It ought to be possible to make migration possible
without knowing the full key ... but it would require that the salt isn't
used in crypto, or is easily reversed out... hmmm.
>
>
> Modified:
branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
> ===
> ---
branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
2008-05-06 07:39:23 UTC (rev 19788)
> +++
branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
2008-05-06 07:39:43 UTC (rev 19789)
> @@ -35,6 +35,8 @@
> * @author sdiz
> */
> public class SaltedHashFreenetStore implements FreenetStore {
> + private static final boolean OPTION_SAVE_PLAINKEY = true;
> +
> private static boolean logMINOR;
> private static boolean logDEBUG;
>
> @@ -182,6 +184,8 @@
>
> /** Flag for occupied space */
> private final long ENTRY_FLAG_OCCUPIED = 0x0001L;
> + /** Flag for plain key available */
> + private final long ENTRY_FLAG_PLAINKEY = 0x0002L;
>
> private static final long ENTRY_HEADER_LENGTH = 0x70L;
> private final long entryPaddingLength;
> @@ -202,8 +206,9 @@
>* ++---+---+
>* |0030| Flag | Store Size |
>* ++---+---+
> - * |0040|Reserved |
> - * |0050|Reserved |
> + * |0040| Plain Routing Key |
> + * |0050| (Only if ENTRY_FLAG_PLAINKEY) |
> + * ++---+
>* |0060|Reserved |
>* ++---+
>* |0070| Encrypted Header|
> @@ -218,7 +223,8 @@
>* written, ignored on read.
>*/
> private class Entry {
> - private byte[] routingKey;
> + private byte[] plainRoutingKey;
> + private byte[] encryptedRoutingKey;
> private byte[] dataEncryptIV;
> private long flag;
> private long storeSize;
> @@ -230,12 +236,13 @@
> /**
>* Create a new entry
>*
> - * @param routingKey
> + * @param plainRoutingKey
>* @param header
>* @param data
>*/
> - public Entry(byte[] routingKey, byte[] header, byte[] data) {
> - this.routingKey = routingKey;
> + public Entry(byte[] plainRoutingKey, byte[] header, byte[]
> data) {
> + this.plainRoutingKey = plainRoutingKey;
> +
> flag = ENTRY_FLAG_OCCUPIED;
> storeSize = SaltedHashFreenetStore.this.storeSize;
>
> @@ -245,6 +252,10 @@
> System.arraycopy(header, 0, this.header, 0,
> headerBlockLength);
> this.data = new byte[dataBlockLength];
> System.arraycopy(data, 0, this.data, 0,
> dataBlockLength);
> +
> + if (OPTION_SAVE_PLAINKEY) {
> + flag |= ENTRY_FLAG_PLAINKEY;
> + }
>
> isEncrypted = false;
> }
> @@ -252,8 +263,8 @@
> public Entry(ByteBuffer in) {
> assert in.remaining() == entryTotalLength;
>
> - routingKey = new byte[0x20];
> - in.get(routingKey);
> + encryptedRoutingKey = new byte[0x20];
> + in.get(encryptedRoutingKey);
>
> dataEncryptIV = new byte[0x10];
> in.get(dataEncryptIV);
> @@ -261,6 +272,11 @@
> flag = in.getLong();
> storeSize = in.getLong();
>
> + if ((flag & ENTRY_FLAG_PLAINKEY) != 0) {
> + plainRoutingKey = new byte[0x20];
> + in.get(plainRoutingKey);
> + }
> +
> // reserved bytes
> in.position((int) ENTRY_HEADER_LENGTH);
>
> @@ -278,12 +294,16 @@
> public ByteBuffer toByteBuffer() {
> ByteBuffer out = ByteBuffer.allocate((int)
> entryTotalLength);
> encrypt();
> - out.put(routingKey);
> + out.put(encryptedRoutingKey);
> out.put(dataEncryptIV);
>
> out.putLong(flag);
>
Re: [freenet-dev] [freenet-cvs] r19733 - branches/saltedhashstore/freenet/src/freenet/store
On Sunday 04 May 2008 14:11, you wrote:
> Author: j16sdiz
> Date: 2008-05-04 13:11:55 + (Sun, 04 May 2008)
> New Revision: 19733
>
> Added:
>
branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
> Log:
> SaltedHashFreenetStore basic function (without locking/resizing)
>
>
> Added:
branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
> ===
> ---
branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
(rev 0)
> +++
branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java
2008-05-04 13:11:55 UTC (rev 19733)
> +
> + public StorableBlock getStorableBlock(byte[] routingKey, byte[]
> fullKey)
throws KeyVerifyException {
> + if (!decrypt(routingKey))
> + return null;
> +
> + StorableBlock block = callback.construct(data, header,
> routingKey,
fullKey);
> + byte[] blockRoutingKey = block.getRoutingKey();
> +
> + if (!Arrays.equals(blockRoutingKey, routingKey)) {
> + // either the data is corrupted or we have
> found a SHA-1 collision
> + // can't recover, as decrypt() depends on a
> correct route key
> + return null;
> + }
> +
> + return block;
> + }
If we can't be sure that the Entry isn't re-encrypted, we should copy the
data/headers here. If we can, we should document it.
> +
> + /**
> + * Check if a block is free
> + *
> + * @param offset
> + * @throws IOException
> + */
> + private boolean isFree(long offset) throws IOException {
> + int split = (int) (offset % FILE_SPLIT);
> + long rawOffset = (offset / FILE_SPLIT) * entryTotalLength;
> +
> + ByteBuffer bf = ByteBuffer.allocate((int) ENTRY_HEADER_LENGTH);
> +
> + do {
> + int status = storeFC[split].write(bf, rawOffset +
> bf.position());
> + if (status == -1)
> + throw new EOFException();
> + } while (bf.hasRemaining());
> +
> + return (bf.getLong(0x30) & ENTRY_FLAG_OCCUPIED) == 0;
> + }
Why read the whole entry just to check one byte (or one long)?
pgpF9rikAKr44.pgp
Description: PGP signature
___
Devl mailing list
Devl@freenetproject.org
http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl
[freenet-dev] [freenet-cvs] r19836 - in trunk/website: . includes pages/de
* Michael T?nzer [2008-05-09 09:32:01]: > Florent Daigni?re schrieb: > > * Michael T?nzer [2008-05-08 23:35:21]: > > > >> Some of your changes broke the link to the English news site on the > >> German one. The old link doesn't work either. > >> We need that link because the news section needs to be up to date and I > >> can't always translate it in real time. > > > > Okay, what's the exact url of the broken link ? > > > > Well it should link to the English news section, before your changes it > was href="/index.php?site=news&lang=en" then you changed it to > href="/news.html?lang=en" neither the old nor the new one work now (one > just gets the german section, because of the content negotiation with > the browser). Fixed :) Please don't use the old links anymore. -- next part -- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 189 bytes Desc: Digital signature URL: <https://emu.freenetproject.org/pipermail/devl/attachments/20080509/2effde97/attachment.pgp>
[freenet-dev] Post-0.7.0 priorities
-BEGIN PGP SIGNED MESSAGE- Hash: SHA1 | Automatic bandwidth calibration. Other p2p apps have this, we should have it. Good idea. Also, we should definitely look into better utilizing available bandwidth. Freenet's the only p2p app which consistently underutilizes my upload limit (~ 2 Mbit/s out of 8 Mbit/s of link capacity). I understand that we don't want to create supernodes, but come on, 2 Mbit/s is *nothing* these days. | Client layer changes: I propose to move the entire client layer onto disk. We I say this deserves to be moved to High or Very High priority. The main problem is not memory usage as is (most people have 1 Gb+ of RAM now), but rather inability of Java to properly grow and shrink its memory usage on the "as needed" basis. Not a single native Windows application behaves this way, and I doubt many users are prepared to understand and adjust memory limits manually. Heck, even I, being a Java developer, spent several days trying to understand what heap limit to set for Freenet so that it won't run out of memory (and that it uses ~ 2x memory on 64-bit JVM doesn't help any). Also, perhaps we can detect OOM errors and offer the user to increase the memory limit next time he tries to run Freenet? | Auto-update for plugins: We should have had this ages ago. Several people have Shouldn't we consider auto-updating bundled applications as well? Or perhaps providing an auto-update API for use by third-party apps? Just a thought. Hope the above was helpful, Regards, Victor Denisov. -BEGIN PGP SIGNATURE- Version: GnuPG v1.4.5 (MingW32) Comment: Using GnuPG with Mozilla - http://enigmail.mozdev.org iD8DBQFII+7pS81Mh9/iCDgRAhN7AJ93mwVa2Ogqo+6JLiY2322AgPm2WwCgwYmw DQ2jkkYDeZ07pXaoE/qH/Uo= =WSBl -END PGP SIGNATURE-
[freenet-dev] [freenet-cvs] r19836 - in trunk/website: . includes pages/de
Florent Daigni?re schrieb: > * Michael T?nzer [2008-05-08 23:35:21]: > >> Some of your changes broke the link to the English news site on the >> German one. The old link doesn't work either. >> We need that link because the news section needs to be up to date and I >> can't always translate it in real time. > > Okay, what's the exact url of the broken link ? > Well it should link to the English news section, before your changes it was href="/index.php?site=news&lang=en" then you changed it to href="/news.html?lang=en" neither the old nor the new one work now (one just gets the german section, because of the content negotiation with the browser). -- Follow the blue rabbit - The Freenet Project - http://freenetproject.org/ -- next part -- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 252 bytes Desc: OpenPGP digital signature URL: <https://emu.freenetproject.org/pipermail/devl/attachments/20080509/50b5cfa2/attachment.pgp>
[freenet-dev] Post 0.7 idea: off-grid darknet!
On Fri, May 9, 2008 at 2:58 AM, Matthew Toseland wrote: > On Thursday 08 May 2008 01:41, Matthew Toseland wrote: >> https://bugs.freenetproject.org/view.php?id=2345 >> >> --- >> A typical domestic internet connection has at most 1Mbps uplink. In some >> megacities 100Mbps or even 1Gbps is available (symmetric), however it is >> unlikely that the bandwidth available in most homes will exceed a few >> megabits in the near future. >> >> We could implement darknet sneakernet connections by exchanging USB sticks. >> E.g. if you meet somebody every day (e.g. a coworker), you could exchange >> (cheap) 8G sticks, plug them in overnight, and then do the same again the >> next day. This would produce approx 100K/sec (1Mbps) each way for each > person >> you did it with. >> >> The performance here imho isn't world-shattering, but nonetheless it's >> interesting. And it would build the darknet, some of it completely off-grid, >> work in many places where nothing else does safely, and get us some great >> headlines. >> --- >> >> IMHO this is something we should seriously consider, if not this year, then > at >> least next year during the 0.8 cycle. The main technical prerequisite is >> token passing load management, unless we implement a completely different >> load management system for it. True passive requests would help in that >> they'd make publish/subscribe work much better on this. Even if it's not >> perfect, it'd be a very interesting way to get people in, and far from being >> a publicity stunt, it would be of immense long-term value. >> > > Ian is of the view that this should be a separate application based on similar > principles to Freenet. I'm not. We agree that there are some significant > issues to deal with. I am of the view that these networks are mutually > complementary and therefore should talk to each other: Darknet over UDP isn't > safe in hostile environments, and off-grid darknets a) work much better if > parts of them are online (certainly we could expect some covert wireless > links in places, but being able to link to a functional on-grid darknet would > surely be a benefit; long links are going to be rare on a pure off-grid > darknet), and b) would be much easier to bootstrap from a working on-grid > darknet. > > W.r.t. code (and to some degree protocol), IMHO most of Freenet's code would > be useful to a darknet sneakernet implementation: > - The entire client layer could be reused. The queue is by definition a long > term structure, Fproxy offers to download stuff in the background and tells > you when it's done etc. FCP could be reused, although on a > pure-off-grid-darknet node, clearly they would have to use it in a different > way to what they do now. > - Full passive requests would be virtually identical for the two > implementations. ULPRs could be adapted without too much difficulty. This > makes FMS etc somewhat feasible, if slow, and anything that can be seen as > publish/subscribe (e.g. getting new editions of freesites) possible. Full > passive requests are a long term goal as they would have some interesting > advantages, but even ULPRs, with sufficient tweaking, may be sufficient to > make this usable. > - The link layer would obviously be worthless, except in the IMHO interesting > case where you have both a darknet connection *and* exchange of flash cards > going on with a peer. Thus low latency requests such as Frost traffic can go > over the link, and when you queue a big splitfile, it would fetch the top > blocks of the pyramid over the link, and then queue the rest to come back > over the following day's card exchange. > - Request priorities would be necessary. > - We probably couldn't reuse the current load limiting/management code. We > would in all likelihood need token passing. This is something we will need > long term anyway, of course. > - Swapping: This is probably the hardest part. Our current strategy involves a > commit/reveal protocol (4 round trips). This clearly won't work well on a > pure off-grid darknet. Doing a large part of the work offline will be > necessary, and to do that a lot of topology may need to be exposed... which > is bad because it makes life easier for a well-resourced attacker. Also, the > off-grid network may have to be partially separate in routing terms through > some sort of tiered routing (look at the network labelling code for something > related). > - User interface to transport: We want users to be able to just plug in a > bunch of USB sticks to a mini-hub, and have Freenet auto-detect that they are > formatted for it, download from them, and then upload to them, all ready for > the next day for them to be swapped back. I don't know what native support > this would require. Binary Blobs (.fblob)? > Based on the above, IMHO this *might* be feasible, it becomes a lot more > interesting after some of the features we have planned for 0.7.1/0.8 have > been implemented, but even then there are some major problems to solve, such >
Re: [freenet-dev] Post-0.7.0 priorities
On Thursday 08 May 2008 23:22, Matthew Toseland wrote: > I've made a bug on the bug tracker to which I've linked all the things that I > think *might* be important for 0.7.1. Please contribute to this bug by > setting it related to anything that you think it should be related to, or > reply to this thread. > > Stuff I think is important for the next release: > > Automatic bandwidth calibration. Other p2p apps have this, we should have it. > It should significantly improve the average output bandwidth available, since > most users presumably don't set the output limit. Further it would improve > usability by making the connection more responsive. > Timescale: Unclear at this point. > Priority: High. > > Datastore changes: It looks very much like we can have both better network > performance and much less memory overhead by using a non-associative salted > hash table on disk. Daniel Cheng (sdiz) is currently implementing this on a > branch. I will be reviewing the code soon. Nextgens has asked about its > suitability for the cache as opposed to the store; mrogers' simulations were > based on it being for the store. > Timescale: A lot of this is done already, most work will be sdiz's. > Priority: High. > > More work on ULPRs: Various changes related to coalescing and temporarily > suppressing requests if there are too many for a single key over a period. > Should help with FMS and to boost payload %. > Timescale: 1 week > Priority: High. > > Use peers' peers' locations for routing: According to oskar and vive, this > should significantly cut average path lengths. There is minimal security > impact as this data is already exposed by swapping. > Timescale: 1 week > Priority: High. > > Faster swapping: Vive has some ideas to improve swapping performance > significantly. Hopefully he can turn these into implementible proposals. This > would significantly improve performance (especially given the level of churn > we see), and also help with security (because we could reset more to prevent > swapping attacks). > Timescale: Depends on Vive. Implementation probably relatively easy. > Priority: High if possible. Would justify calling it 0.8.0 IMHO. > > Streams and MTU: We can improve our payload proportion significantly, allow > for much smaller packets, support smaller MTUs, avoid padding with random > data, and support simple stego, by implementing transport layer streams. > There are also a number of other mostly minor changes which we should > implement to make Freenet work well on low MTU connections. > Timescale: 2-4 weeks?? > Priority: High. > > Better Librarian integration: We should have a search box on the homepage, it > should be embeddable in freesites. And XMLSpider probably needs some more > debugging. > Timescale: 1 week or less. > Priority: High. > > Client layer changes: I propose to move the entire client layer onto disk. We > would continue to store enough data to restart requests from scratch in > downloads.dat.gz, but we would have an on-disk queue structure instead of an > in-memory queue structure. This combined with the datastore changes would > make our memory usage fixed, regardless of the size of your store or the > number of requests you queue. It would solve many bug reports, it would > incorporate the long-awaited true request resuming, would make Freenet run > well on home servers with 128M or maybe even 64M of RAM, and generally is a > good idea. > Timescale: 2 weeks?? > Priority: Medium. > > Auto-update for plugins: We should have had this ages ago. Several people have > been complaining about it, it is a security issue if you want them kept up to > date. It shouldn't be difficult. > Timescale: 3 days > Priority: Medium. > > Better content filter: Filter on insert by default (for performance on fetch), > support more HTML etc, support RSS, support some other formats (e.g. mp3), > etc. > Timescale: 1 week, more for more formats. > Priority: Medium. > > Better USKs: Hierarchical DBRs would help USK lookups to find something close > to the latest version quickly, then plod through the editions from there to > find the latest version. > Timescale: 2 days. > Priority: Medium. > > System tray icon: IMHO this would be a good usability feature. It would make > it obvious that Freenet is running in the background and make it easy to > throttle it or disable it when you want to play an MMORPG etc. > Timescale: 1 week??? > Priority: Medium. > > Bandwidth: mister_wavey is working on a bandwidth scheduler. It would be a big > help for a lot of users who have off-peak quotas etc. There will be a user > friendly config sub-page for it. Average bandwidth limits, maybe actually > telling the node a monthly quota, would also be useful for many users. More > accurate bandwidth limiting (limiting all packets not just transfers) would > also be a big help for users on slower upstream connections. > Timescale: Unknown. > Priority: Medium
Re: [freenet-dev] Post-0.7.0 priorities
On Friday 09 May 2008 07:27, Victor Denisov wrote: > | Automatic bandwidth calibration. Other p2p apps have this, we should > have it. > > Good idea. Also, we should definitely look into better utilizing > available bandwidth. Freenet's the only p2p app which consistently > underutilizes my upload limit (~ 2 Mbit/s out of 8 Mbit/s of link > capacity). I understand that we don't want to create supernodes, but > come on, 2 Mbit/s is *nothing* these days. IMHO automatic bandwidth calibration will help a lot with this. Beyond that we're looking at token passing, which may be too big for 0.7.1. > > | Client layer changes: I propose to move the entire client layer onto > disk. We > > I say this deserves to be moved to High or Very High priority. The main > problem is not memory usage as is (most people have 1 Gb+ of RAM now), > but rather inability of Java to properly grow and shrink its memory > usage on the "as needed" basis. Not a single native Windows application > behaves this way, and I doubt many users are prepared to understand and > adjust memory limits manually. Heck, even I, being a Java developer, > spent several days trying to understand what heap limit to set for > Freenet so that it won't run out of memory (and that it uses ~ 2x memory > on 64-bit JVM doesn't help any). Also, perhaps we can detect OOM errors > and offer the user to increase the memory limit next time he tries to > run Freenet? Agreed, memory usage is a usability issue: the user shouldn't have to care about it. > > | Auto-update for plugins: We should have had this ages ago. Several > people have > > Shouldn't we consider auto-updating bundled applications as well? Or > perhaps providing an auto-update API for use by third-party apps? Just a > thought. Maybe, that would be harder though. I would be happy to discuss it with their authors. > > Hope the above was helpful, > > Regards, > Victor Denisov. pgpcz1OB58GLH.pgp Description: PGP signature ___ Devl mailing list Devl@freenetproject.org http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl
Re: [freenet-dev] What to do with the 0.5 related stuffs?
* Ringo Kamens <[EMAIL PROTECTED]> [2008-05-08 23:35:20]: > I think we should keep .5 stuff around, since a lot of things link to > it We can provide permanent redirects to new content, that's not a problem. > but there's no reason to advertise it. Also, I get the feeling that > lots of freenet users don't trust .7 yet and that the .5 community > will continue to flourish for quite some time. Where are those users? that's the big question... They don't use our infrastructure anymore that's for sure. The number of hits on the .5 pages is ridiculous; We don't get much downloads either... The only service some are still using is the seednode-generation one. signature.asc Description: Digital signature ___ Devl mailing list Devl@freenetproject.org http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl
Re: [freenet-dev] Content Management System
* Michael Tänzer <[EMAIL PROTECTED]> [2008-05-08 22:47:28]: > Ian Clarke schrieb: > > You could argue that all of those things have it easy, because most > > people understand what those things do, they don't need an elaborate > > explanation. But look at Gnome's website: > > > > http://www.gnome.org/ > > > > It is clean, simple, yet if you need to you can quickly dig down to a > > vast wealth of information. > > > > Gnome uses Plone as it's CMS. This may be not the best choice for us > though, as Plone is Phyton based, but that's something nextgens might > know better than me. > I don't have any problem with using python; Trac is in python too iirc... But I don't think that Plone is what we need, it's a framework more than a CMS. > > Either way, since we are Java hackers for the most part, not web > > designers, I strongly recommend that we borrow as much as we can from > > elsewhere, even so far as using free or creative commons HTML and CSS > > verbatim, perhaps with only a few minor changes. > > What about java-based CMSes then ? :p signature.asc Description: Digital signature ___ Devl mailing list Devl@freenetproject.org http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl
[freenet-dev] [freenet-cvs] r19836 - in trunk/website: . includes pages/de
* Michael T?nzer [2008-05-08 23:35:21]: > Some of your changes broke the link to the English news site on the > German one. The old link doesn't work either. > We need that link because the news section needs to be up to date and I > can't always translate it in real time. Okay, what's the exact url of the broken link ? -- next part -- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 189 bytes Desc: Digital signature URL: <https://emu.freenetproject.org/pipermail/devl/attachments/20080509/25e7f1ad/attachment.pgp>
[freenet-dev] What to do with the 0.5 related stuffs?
Hi, I'm currently in the process of cleaning up our server... And I'm planning to get rid of all the .5-related stuffs now that 0.7 is officially released. That includes our builds and seednode files hosted on downloads.freenetproject.org, a *lot* of apache redirection for old URLs and the short mention of it on the download page of our website. Obviously we will keep the source code in our svn-tree. Any objection? If so, object soon or never. NextGen$ -- next part -- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 189 bytes Desc: Digital signature URL: <https://emu.freenetproject.org/pipermail/devl/attachments/20080509/77ef4ec8/attachment.pgp>
[freenet-dev] Post 0.7 idea: off-grid darknet!
or a well-resourced attacker. Also, the > > off-grid network may have to be partially separate in routing terms through > > some sort of tiered routing (look at the network labelling code for something > > related). > > - User interface to transport: We want users to be able to just plug in a > > bunch of USB sticks to a mini-hub, and have Freenet auto-detect that they are > > formatted for it, download from them, and then upload to them, all ready for > > the next day for them to be swapped back. I don't know what native support > > this would require. > > Binary Blobs (.fblob)? > > > Based on the above, IMHO this *might* be feasible, it becomes a lot more > > interesting after some of the features we have planned for 0.7.1/0.8 have > > been implemented, but even then there are some major problems to solve, such > > as swapping. > > I think something like Freenet over HTTP should have higher priority. > There are lots of user behind HTTP/HTTPS-only proxy. Mostly using it at work when they shouldn't be. :) Well I suppose some students too... But Freenet over HTTP is a conventional application of transport plugins, so shouldn't be too much of a problem once we have them. -- next part -- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available URL: <https://emu.freenetproject.org/pipermail/devl/attachments/20080509/4ae41937/attachment.pgp>
Re: [freenet-dev] [freenet-cvs] r19836 - in trunk/website: . includes pages/de
* Michael Tänzer <[EMAIL PROTECTED]> [2008-05-09 09:32:01]: > Florent Daignière schrieb: > > * Michael Tänzer <[EMAIL PROTECTED]> [2008-05-08 23:35:21]: > > > >> Some of your changes broke the link to the English news site on the > >> German one. The old link doesn't work either. > >> We need that link because the news section needs to be up to date and I > >> can't always translate it in real time. > > > > Okay, what's the exact url of the broken link ? > > > > Well it should link to the English news section, before your changes it > was href="/index.php?site=news&lang=en" then you changed it to > href="/news.html?lang=en" neither the old nor the new one work now (one > just gets the german section, because of the content negotiation with > the browser). Fixed :) Please don't use the old links anymore. signature.asc Description: Digital signature ___ Devl mailing list Devl@freenetproject.org http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl
Re: [freenet-dev] [freenet-cvs] r19836 - in trunk/website: . includes pages/de
Florent Daignière schrieb: > * Michael Tänzer <[EMAIL PROTECTED]> [2008-05-08 23:35:21]: > >> Some of your changes broke the link to the English news site on the >> German one. The old link doesn't work either. >> We need that link because the news section needs to be up to date and I >> can't always translate it in real time. > > Okay, what's the exact url of the broken link ? > Well it should link to the English news section, before your changes it was href="/index.php?site=news&lang=en" then you changed it to href="/news.html?lang=en" neither the old nor the new one work now (one just gets the german section, because of the content negotiation with the browser). -- Follow the blue rabbit - The Freenet Project - http://freenetproject.org/ signature.asc Description: OpenPGP digital signature ___ Devl mailing list Devl@freenetproject.org http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl
