Author: jflesch
Date: 2006-10-29 15:26:56 +0000 (Sun, 29 Oct 2006)
New Revision: 10735
Modified:
trunk/apps/Thaw/src/thaw/core/PluginManager.java
trunk/apps/Thaw/src/thaw/fcp/FCPClientGet.java
trunk/apps/Thaw/src/thaw/plugins/index/Index.java
Log:
Use streams instead of a temporary file when fetching index
Modified: trunk/apps/Thaw/src/thaw/core/PluginManager.java
===================================================================
--- trunk/apps/Thaw/src/thaw/core/PluginManager.java 2006-10-29 15:08:53 UTC
(rev 10734)
+++ trunk/apps/Thaw/src/thaw/core/PluginManager.java 2006-10-29 15:26:56 UTC
(rev 10735)
@@ -12,8 +12,8 @@
"thaw.plugins.InsertPlugin",
"thaw.plugins.FetchPlugin",
"thaw.plugins.StatusBar",
-
"thaw.plugins.IndexEditor",
-
"thaw.plugins.IndexBrowser"};
+
"thaw.plugins.IndexBrowser",
+
"thaw.plugins.IndexEditor"};
private Core core = null;
Modified: trunk/apps/Thaw/src/thaw/fcp/FCPClientGet.java
===================================================================
--- trunk/apps/Thaw/src/thaw/fcp/FCPClientGet.java 2006-10-29 15:08:53 UTC
(rev 10734)
+++ trunk/apps/Thaw/src/thaw/fcp/FCPClientGet.java 2006-10-29 15:26:56 UTC
(rev 10735)
@@ -2,6 +2,10 @@
import java.io.File;
import java.io.FileOutputStream;
+import java.io.PipedInputStream;
+import java.io.PipedOutputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
import java.util.Observer;
import java.util.Observable;
@@ -45,7 +49,9 @@
private boolean alreadySaved = false;
+ private PipedOutputStream pipedOutputStream = new PipedOutputStream();
+
/**
* See setParameters().
*/
@@ -69,6 +75,7 @@
/**
* Used to resume query from persistent queue of the node.
* Think of adding this FCPClientGet as a queryManager observer.
+ * @param destinationDir if null, then you are expected to use the
streams (see getInputStream())
*/
public FCPClientGet(String id, String key, int priority,
int persistence, boolean globalQueue,
@@ -107,7 +114,7 @@
/**
* Only for initial queries : To resume queries, use
FCPClientGet(FCPQueueManager, Hashmap).
- * @param destinationDir if null, won't be automatically saved
+ * @param destinationDir if null, you're expected to use the streams
* @param persistence 0 = Forever ; 1 = Until node reboot ; 2 = Until
the app disconnect
*/
public FCPClientGet(String key, int priority,
@@ -393,7 +400,7 @@
this.notifyObservers();
- if(this.fetchDirectly(queryManager.getConnection(),
this.getPath(), this.fileSize, true)) {
+ if(this.fetchDirectly(queryManager.getConnection(),
this.fileSize, true)) {
this.successful = true;
this.status = "Available";
} else {
@@ -583,21 +590,29 @@
}
+ private boolean fetchDirectly(FCPConnection connection, long size,
boolean reallyWrite) {
+ String file = this.getPath();
+ File newFile = null;
+ OutputStream outputStream = null;
- private boolean fetchDirectly(FCPConnection connection, String file,
long size, boolean reallyWrite) {
- File newFile = new File(file);
- FileOutputStream fileWriter = null;
+ if (file != null) {
+ newFile = new File(file);
+ }
-
if(reallyWrite) {
- Logger.info(this, "Writing file to disk ...
('"+file+"')");
+ Logger.info(this, "Getting file from node ... ");
- try {
- fileWriter = new FileOutputStream(newFile);
- } catch(java.io.IOException e) {
- Logger.error(this, "Unable to write file on
disk ... disk space / perms ? : "+e.toString());
- this.status = "Write error";
- return false;
+ if (newFile != null) {
+ try {
+ outputStream = new
FileOutputStream(newFile);
+ } catch(java.io.IOException e) {
+ Logger.error(this, "Unable to write
file on disk ... disk space / perms ? : "+e.toString());
+ this.status = "Write error";
+ return false;
+ }
+ } else {
+ Logger.info(this, "Use PipedOutputStream");
+ outputStream = pipedOutputStream;
}
} else {
Logger.info(this, "File is supposed already written.
Not rewriting.");
@@ -632,7 +647,7 @@
if(reallyWrite) {
try {
- fileWriter.write(read, 0, amount);
+ outputStream.write(read, 0, amount);
} catch(java.io.IOException e) {
Logger.error(this, "Unable to write
file on disk ... out of space ? : "+e.toString());
this.status = "Write error";
@@ -658,9 +673,9 @@
if(reallyWrite) {
try {
- fileWriter.close();
+ outputStream.close();
- if(!writingSuccessful)
+ if(!writingSuccessful && newFile != null)
newFile.delete();
} catch(java.io.IOException e) {
@@ -675,6 +690,17 @@
}
+ public InputStream getInputStream() {
+ try {
+ return new PipedInputStream(pipedOutputStream);
+ } catch(java.io.IOException e) {
+ Logger.error(this, "Error while instanciating
PipedInputStream: "+e.toString());
+ return null;
+ }
+ }
+
+
+
public boolean removeRequest() {
FCPMessage stopMessage = new FCPMessage();
Modified: trunk/apps/Thaw/src/thaw/plugins/index/Index.java
===================================================================
--- trunk/apps/Thaw/src/thaw/plugins/index/Index.java 2006-10-29 15:08:53 UTC
(rev 10734)
+++ trunk/apps/Thaw/src/thaw/plugins/index/Index.java 2006-10-29 15:26:56 UTC
(rev 10735)
@@ -22,8 +22,12 @@
import org.w3c.dom.NodeList;
import java.io.OutputStream;
+import java.io.PipedOutputStream;
+import java.io.InputStream;
+import java.io.PipedInputStream;
import java.io.FileOutputStream;
+
import thaw.fcp.*;
import thaw.plugins.Hsqldb;
import thaw.core.*;
@@ -237,8 +241,13 @@
public void update() {
- this.targetFile = new java.io.File(this.toString()+".xml");
+ String tmpdir = System.getProperty("java.io.tmpdir");
+ if (tmpdir == null)
+ tmpdir = "";
+
+ this.targetFile = new java.io.File(tmpdir +
java.io.File.separator + this.toString()+".xml");
+
if (this.transfer != null) {
Logger.notice(this, "A transfer is already running");
return;
@@ -248,8 +257,18 @@
FCPClientPut clientPut;
Logger.info(this, "Generating index ...");
- this.generateXML(this.targetFile);
+ FileOutputStream outputStream;
+
+ try {
+ outputStream = new FileOutputStream(targetFile);
+ } catch(java.io.FileNotFoundException e) {
+ Logger.warning(this, "Unable to create file
'"+targetFile.toString()+"' ! not generated !");
+ return;
+ }
+
+ this.generateXML(outputStream);
+
if(this.targetFile.exists()) {
Logger.info(this, "Inserting new version");
@@ -303,9 +322,8 @@
Logger.info(this, "Key asked: "+key);
- clientGet = new FCPClientGet(key, 2, 2, false, -1,
System.getProperty("java.io.tmpdir"));
+ clientGet = new FCPClientGet(key, 2, 2, false, -1, null);
this.transfer = clientGet;
- clientGet.addObserver(this);
/*
* These requests are usually quite fast, and don't consume a
lot
@@ -315,6 +333,9 @@
//this.queueManager.addQueryToThePendingQueue(clientGet);
clientGet.start(queueManager);
+ Thread downloadAndParse = new Thread(new
DownloadAndParse(clientGet, rewriteKey));
+ downloadAndParse.start();
+
this.setChanged();
this.notifyObservers();
}
@@ -324,6 +345,34 @@
}
+ private class DownloadAndParse implements Runnable {
+ FCPClientGet clientGet;
+ boolean rewriteKey;
+
+ public DownloadAndParse(FCPClientGet clientGet, boolean
rewriteKey) {
+ this.clientGet = clientGet;
+ }
+
+ public void run() {
+ loadXML(clientGet.getInputStream());
+
+ if (rewriteKey)
+ publicKey = transfer.getFileKey();
+ else
+ revision =
FreenetURIHelper.getUSKRevision(transfer.getFileKey());
+
+ Logger.info(this, "Most up-to-date key found: " +
publicKey);
+
+ save();
+
+ transfer = null;
+
+ setChanged();
+ notifyObservers();
+ }
+ }
+
+
protected void setTransfer(FCPTransferQuery query) {
this.transfer = query;
@@ -509,33 +558,6 @@
return;
}
- if(this.transfer instanceof FCPClientGet) {
- java.io.File file = new
java.io.File(this.transfer.getPath());
-
- Logger.info(this, "Updating index ...");
-
- if (this.rewriteKey)
- this.publicKey =
this.transfer.getFileKey();
- else
- this.revision =
FreenetURIHelper.getUSKRevision(this.transfer.getFileKey());
-
- Logger.info(this, "Most up-to-date key
found: " + this.publicKey);
-
- this.loadXML(file);
- this.save();
-
- Logger.info(this, "Update done.");
-
- file.delete();
-
- this.transfer = null;
-
- this.setChanged();
- this.notifyObservers();
-
- return;
- }
-
}
if (this.transfer.isFinished() &&
!this.transfer.isSuccessful()) {
@@ -846,26 +868,6 @@
//// XML ////
- public void generateXML(java.io.File file) {
-
- FileOutputStream outputStream;
-
- try {
- outputStream = new FileOutputStream(file);
- } catch(java.io.FileNotFoundException e) {
- Logger.warning(this, "Unable to create file '"+file+"'
! not generated !");
- return;
- }
-
- this.generateXML(outputStream);
-
- try {
- outputStream.close();
- } catch(java.io.IOException e) {
- Logger.error(this, "Unable to close stream because:
"+e.toString());
- }
- }
-
public void generateXML(OutputStream out) {
StreamResult streamResult = new StreamResult(out);
@@ -983,7 +985,7 @@
return files;
}
- public void loadXML(java.io.File file) {
+ public void loadXML(java.io.InputStream input) {
DocumentBuilderFactory xmlFactory =
DocumentBuilderFactory.newInstance();
DocumentBuilder xmlBuilder;
@@ -997,7 +999,7 @@
Document xmlDoc;
try {
- xmlDoc = xmlBuilder.parse(file);
+ xmlDoc = xmlBuilder.parse(input);
} catch(org.xml.sax.SAXException e) {
Logger.error(this, "Unable to load index because:
"+e.toString());
return;
