Well, this is what it says in the subject line - a patch which implents a storePercent config option using a ClientDiskFree FCP command.
I included a diskSpace.py which is a python script for unix that does
the required native magic on unix for this to work - if someone could
add this functionality into the widnwos minder app, that would be
useful, as I don't have the required Visual C++ (I only have mingw,
which doens't seem to be sufficient to compile NodeConfig).
Side note: the percentage of free disk calculation includes the
data int ehs tore - so, on my drive with 540meg free + 230meg in the store,
i get a 391meg storeSize. If storeSize would be less than the config file
storeSize, is it set to that (so storeSize is now the *minimum* store size).
This relies on NativeFSDir - i assume that no-one is using the monolithic
store aymore.
- fish
Index: src/freenet/fs/dir/NativeFSDirectory.java
===================================================================
RCS file: /cvsroot/freenet/freenet/src/freenet/fs/dir/NativeFSDirectory.java,v
retrieving revision 1.135
diff -u -3 -p -u -r1.135 NativeFSDirectory.java
--- src/freenet/fs/dir/NativeFSDirectory.java 22 Jun 2003 07:56:51 -0000 1.135
+++ src/freenet/fs/dir/NativeFSDirectory.java 25 Jun 2003 15:15:02 -0000
@@ -640,7 +640,7 @@ public class NativeFSDirectory implement
public final File root;
public final String rootAsString;
- public final long size;
+ public long size; // no long final to allow for resizing
private final int maxFilenameBaseLength;
private final StringBuffer getFileBuffer;
// GC takes precedence over contention because getFile() is short
@@ -782,6 +782,14 @@ public class NativeFSDirectory implement
System.runFinalization();
Core.logger.log(this, "Currently used memory (started NativeFSDirectory, after
GC, "+buffers.size()+" keys): "+(Runtime.getRuntime().totalMemory() -
Runtime.getRuntime().freeMemory()), Core.logger.DEBUG);
+ }
+
+ // Checking this is the job of someone else ;).
+ // This does what it says on the box... sets the size of the
+ // datastore to size
+ public void setSize(long size)
+ {
+ this.size=size;
}
/** (De-)Allocate space for a temp file write
Index: src/freenet/node/Main.java
===================================================================
RCS file: /cvsroot/freenet/freenet/src/freenet/node/Main.java,v
retrieving revision 1.216
diff -u -3 -p -u -r1.216 Main.java
--- src/freenet/node/Main.java 23 Jun 2003 09:24:22 -0000 1.216
+++ src/freenet/node/Main.java 25 Jun 2003 15:15:09 -0000
@@ -68,7 +68,6 @@ public class Main {
static File tempDir = null;
static FileLoggerHook loggerHook = null;
static LossyDirectory dsDir = null;
- static long storeSize = -1; // Node.storeSize is per store. Of course we only
have one store now...
public static String paramFile;
static NodeConfigUpdater configUpdater = null;
@@ -77,9 +76,17 @@ public class Main {
}
static public long storeSize() {
- return storeSize;
+ return node.storeSize;
+ }
+
+ static public long minStoreSize() {
+ return node.minStoreSize;
}
+ static public long storePercent() {
+ return node.storePercent;
+ }
+
public static NodeConfigUpdater getConfigUpdater() {
return configUpdater;
}
@@ -414,7 +421,6 @@ public class Main {
if(storeFiles[i].exists()) firstTime = false;
}
Directory dir = null;
- storeSize = Node.storeSize;
if ((Node.routingDir == null || Node.routingDir.toString().equals(""))
&& storeFiles[0] != null && !storeFiles[0].equals(""))
@@ -426,7 +432,7 @@ public class Main {
if(storeFiles[0].isDirectory()) {
Node.storeType = "native";
}
- storeSize *= storeFiles.length;
+ node.storeSize *= storeFiles.length;
}
if (!(Node.storeType.equals("monolithic") ||
Node.storeType.equals("native") ||
@@ -469,7 +475,7 @@ public class Main {
// FIXME: replace literals with config values
//
dir = new FSDirectory(fs, new SHA1(), 10000, 10000, 1000, 100);
- storeSize = fs.size();
+ node.storeSize = fs.size();
if (!Node.storeType.equals("convert")) {
Core.logger.log(Main.class, "Monolithic datastore no longer
supported. Converting to native. You had better have enough disk space for the
conversion. If you don't, remove the old storefile.", Core.logger.ERROR);
Node.storeType = "convert";
@@ -482,7 +488,7 @@ public class Main {
try {
newDir = new
NativeFSDirectory(new File(name),
- storeSize,
+ node.storeSize,
Node.storeBlockSize,
Node.useDSIndex,
Node.storeMaxTempFraction);
@@ -549,7 +555,7 @@ public class Main {
dsDir = new LossyDirectory(0x0001, dir);
dsDir.forceFlush();
- DataStore ds = new FSDataStore(dsDir, storeSize / 100);
+ DataStore ds = new FSDataStore(dsDir, node.storeSize / 100);
// load RT
Core.logger.log(Main.class, "loading routing table",
@@ -1898,6 +1904,7 @@ public class Main {
// FCP messages
mh.addType( FCPRawMessage.class, ClientHello.messageName,
ClientHello.class );
mh.addType( FCPRawMessage.class, ClientInfo.messageName, ClientInfo.class
);
+ mh.addType( FCPRawMessage.class, ClientDiskFree.messageName,
ClientDiskFree.class );
mh.addType( FCPRawMessage.class, GenerateSVKPair.messageName,
GenerateSVKPair.class );
mh.addType( FCPRawMessage.class, InvertPrivateKey.messageName,
InvertPrivateKey.class );
mh.addType( FCPRawMessage.class, GenerateCHK.messageName,
GenerateCHK.class );
Index: src/freenet/node/Node.java
===================================================================
RCS file: /cvsroot/freenet/freenet/src/freenet/node/Node.java,v
retrieving revision 1.168
diff -u -3 -p -u -r1.168 Node.java
--- src/freenet/node/Node.java 20 Jun 2003 16:25:37 -0000 1.168
+++ src/freenet/node/Node.java 25 Jun 2003 15:15:17 -0000
@@ -5,7 +5,7 @@ import freenet.crypt.*;
import freenet.config.*;
import freenet.support.*;
import freenet.support.io.*;
-import freenet.fs.dir.Directory;
+import freenet.fs.dir.*;
import freenet.node.ds.*;
import freenet.node.rt.*;
import freenet.presentation.FreenetProtocol;
@@ -87,6 +87,7 @@ public class Node extends Core {
config.addOption("storeSize", 1, "256M", 1010, true); // 256MB is
reasonable, strictish minimum would be 101MB ((1MB chunk + header) * 100)
config.addOption("storeBlockSize", 1, 4096, 1011);
config.addOption("storeMaxTempFraction", 1, (1F/3F), 1012);
+ config.addOption("storePercent", 1, 25, 1013); // use up to 25% of
free disk space
config.addOption("storeCipherName", 1, "Twofish", 1020); // Twofish
cipher
config.addOption("storeCipherWidth", 1, 128, 1021); // 128 bits
config.addOption("routingDir", 1, "", 1022);
@@ -395,6 +396,15 @@ public class Node extends Core {
"Note that if you increase settings such as maxThreads, you
may need to",
"use a larger store.");
+ // storePercent
+ config.argDesc ("storePercent", "<percentage>");
+ config.shortDesc ("storePercent", "Amount of free disk space the datastore
should fill");
+ config.longDesc ("storePercent",
+ "The percentage of the free disk space that the store should
grow to fill.",
+ "This requires the help of a native helper daemon, which
periodically writes",
+ "to <storeDir>/freeSpace. The datasore will never shrink
below storeSize,",
+ "however.");
+
// storeType
config.setExpert ("storeType", true);
config.argDesc ("storeType", "<string>");
@@ -1600,6 +1610,8 @@ public class Node extends Core {
// node limits
static public long storeSize;
+ static public long minStoreSize;
+ static public int storePercent;
static public int storeBlockSize;
static public float storeMaxTempFraction;
static public int rtMaxRefs, rtMaxNodes;
@@ -1662,6 +1674,18 @@ public class Node extends Core {
}
}
+ public void setDiskFree(long diskFree) {
+ if( (dir instanceof NativeFSDirectory))
+ {
+ long usedSpace=storeSize-dir.available();
+ long realDiskFree=diskFree+usedSpace;
+ long nss=(realDiskFree*storePercent)/100;
+ if(nss<minStoreSize)
+ nss=minStoreSize;
+ storeSize=nss;
+ ((NativeFSDirectory)dir).setSize(storeSize);
+ }
+ }
/**
* @throws CoreException if initialization has already occurred
@@ -1746,12 +1770,14 @@ public class Node extends Core {
ThrottledOutputStream.setThrottle(obw);
// set storage parameters
- storeSize = params.getLong("storeSize");
+ minStoreSize = storeSize = params.getLong("storeSize");
if(storeSize < (101L * (1<<20))) {
String error = "Store size insufficient to store 1MB chunks! Your
datastore is so small that it will not be able to store 1MB chunks, the maximum size
of a single data key that most tools insert. You will still be able to fetch them but
your node will not be very useful to the network, and consequentially will not perform
well. To eliminate this error increase your storeSize to at least 101M. It is
currently "+storeSize+".";
System.err.println(error);
Core.logger.log(Node.class, error, Core.logger.ERROR);
}
+ // FIXME: turn this on... when i wrk out wtf is cuasing the exception
+ storePercent = params.getInt("storePercent");
storeBlockSize = params.getInt("storeBlockSize");
storeMaxTempFraction = params.getFloat("storeMaxTempFraction");
//maxFileSize = cacheSize / params.getInt("storeCacheCount");
--- original/./src/freenet/message/client/ClientDiskFree.java 1970-01-01
10:00:00.000000000 +1000
+++ ./src/freenet/message/client/ClientDiskFree.java 2003-06-26 00:28:36.000000000
+1000
@@ -0,0 +1,39 @@
+package freenet.message.client;
+
+import freenet.*;
+import freenet.node.State;
+import freenet.node.states.FCP.*;
+
+/** This is for the FCP handshake.
+ */
+public class ClientDiskFree extends ClientMessage {
+
+ public static final String messageName = "ClientDiskFree";
+ public long diskFree;
+
+ public ClientDiskFree(ConnectionHandler source, RawMessage raw) {
+ super(source, raw);
+ if (!formatError) {
+ try {
+ String diskFreeAsString = otherFields.get("DiskFree");
+ if (diskFreeAsString != null) {
+ diskFree = Long.parseLong(diskFreeAsString, 16);
+ otherFields.remove("DiskFree");
+ }
+ }
+ catch (Exception e) {
+ formatError = true;
+ }
+ }
+ }
+
+ public State getInitialState() {
+ return formatError
+ ? (State) new NewIllegal(id, source, "Error parsing ClientDiskFree
message.")
+ : (State) new NewDiskFree(id, source);
+ }
+
+ public String getMessageName() {
+ return messageName;
+ }
+}
--- original/src/freenet/message/client/NodeDiskFree.java 1970-01-01
10:00:00.000000000 +1000
+++ ./src/freenet/message/client/NodeDiskFree.java 2003-06-25 23:49:03.000000000
+1000
@@ -0,0 +1,18 @@
+package freenet.message.client;
+
+import freenet.FieldSet;
+
+/** This is the FCP handshake.
+ */
+public class NodeDiskFree extends ClientMessage {
+
+ public static final String messageName = "NodeDiskFree";
+
+ public NodeDiskFree(long id, FieldSet fs) {
+ super(id, fs);
+ }
+
+ public String getMessageName() {
+ return messageName;
+ }
+}
--- original/src/freenet/node/states/FCP/NewDiskFree.java 1970-01-01
10:00:00.000000000 +1000
+++ ./src/freenet/node/states/FCP/NewDiskFree.java 2003-06-25 23:48:06.000000000
+1000
@@ -0,0 +1,40 @@
+/* -*- Mode: java; c-basic-indent: 4; tab-width: 4 -*- */
+package freenet.node.states.FCP;
+
+import freenet.*;
+import freenet.node.*;
+import freenet.support.*;
+import freenet.support.io.*;
+import freenet.message.client.*;
+import freenet.fs.dir.NativeFSDirectory;
+import freenet.transport.tcpAddress;
+import freenet.diagnostics.Diagnostics;
+import freenet.node.ds.FSDataStore;
+
+import java.util.*;
+
+public class NewDiskFree extends NewClientRequest {
+
+ public NewDiskFree(long id, ConnectionHandler source) {
+ super(id, source);
+ }
+
+ public final String getName() {
+ return "New Client DiskFree";
+ }
+
+ public State received(Node n, MessageObject mo) throws BadStateException {
+ if (!(mo instanceof ClientDiskFree))
+ throw new BadStateException("expecting ClientDiskFree");
+ FieldSet fs = new FieldSet();
+ ClientDiskFree mdf=(ClientDiskFree)mo;
+
+ // set the store size in the node
+ n.setDiskFree(mdf.diskFree);
+
+ fs.add("StoreSize", Fields.longToString((long)(n.storeSize)));
+ sendMessage(new NodeDiskFree(id, fs));
+ return null;
+ }
+}
+
#!/usr/bin/python
import socket, os, string, statvfs, time
def myHex(data):
return string.lower(hex(long(data))[2:-1])
def openSpecificFcpPort(host, port):
# open the socket
s=None
try:
s=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
mytuple=(host, int(port))
s.connect(mytuple)
return s
except:
s=None
def setNodeDiskSpace(space):
dataString="\x00\x00\x00\x02"
dataString=dataString+"ClientDiskFree\n"
dataString=dataString+"DiskFree="+myHex(space)+"\n"
dataString=dataString+"EndMessage\n"
s=openSpecificFcpPort('127.0.0.1', 8481)
if(s==None):
return 0
s.send(dataString)
dataBack='fish'
cbuf=''
while (len(dataBack)>0):
try:
dataBack=s.recv(1024)
except:
return None
cbuf=cbuf+dataBack
# we have a complete message - process it...
if(cbuf[-11:] == "EndMessage\n"):
lines=string.split(cbuf, "\n")
for j in lines:
fields=string.split(j, "=")
if(fields[0]=="StoreSize"):
return int(fields[1], 16)
return 0
while 0==0:
t=os.statvfs("store/")
freeSpace=(t[statvfs.F_BAVAIL]*t[statvfs.F_FRSIZE])
setNodeDiskSpace(freeSpace)
time.sleep(10)
pgp00000.pgp
Description: PGP signature
