Revision: 6511 Author: sco...@google.com Date: Wed Oct 28 09:26:56 2009 Log: IO improvements to DiskCache and UnifiedAST.
- Renamed "writeTo" to "transferToStream" to avoid the confusion that it actually does the opposite of the other "write" methods, and added its inverse "transferFromStream". - Factored out moveToEndPosition() which fixes a bug where atEnd would *never* be true, forcing an unnecessary seek on every write. - Updated some call sites to take advantage of transferFromStream(), which is more efficient than buffering the whole data stream into a byte buffer. - Updated UnifiedAST to take advantage of DiskCache and the newer serialization techniques, which should reduce memory usage. Review by: spoon http://code.google.com/p/google-web-toolkit/source/detail?r=6511 Modified: /trunk/dev/core/src/com/google/gwt/core/ext/linker/SyntheticArtifact.java /trunk/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardGeneratedResource.java /trunk/dev/core/src/com/google/gwt/dev/jjs/UnifiedAst.java /trunk/dev/core/src/com/google/gwt/dev/util/DiskCache.java ======================================= --- /trunk/dev/core/src/com/google/gwt/core/ext/linker/SyntheticArtifact.java Wed May 20 14:09:16 2009 +++ /trunk/dev/core/src/com/google/gwt/core/ext/linker/SyntheticArtifact.java Wed Oct 28 09:26:56 2009 @@ -19,10 +19,8 @@ import com.google.gwt.core.ext.TreeLogger; import com.google.gwt.core.ext.UnableToCompleteException; import com.google.gwt.dev.util.DiskCache; -import com.google.gwt.dev.util.Util; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.ObjectInputStream; @@ -65,19 +63,17 @@ @Override public void writeTo(TreeLogger logger, OutputStream out) throws UnableToCompleteException { - diskCache.writeTo(token, out); + diskCache.transferToStream(token, out); } private void readObject(ObjectInputStream stream) throws IOException, ClassNotFoundException { stream.defaultReadObject(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - Util.copyNoClose(stream, baos); - token = diskCache.writeByteArray(baos.toByteArray()); + token = diskCache.transferFromStream(stream); } private void writeObject(ObjectOutputStream stream) throws IOException { stream.defaultWriteObject(); - diskCache.writeTo(token, stream); + diskCache.transferToStream(token, stream); } } ======================================= --- /trunk/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardGeneratedResource.java Tue Oct 27 17:23:14 2009 +++ /trunk/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardGeneratedResource.java Wed Oct 28 09:26:56 2009 @@ -20,11 +20,11 @@ import com.google.gwt.core.ext.UnableToCompleteException; import com.google.gwt.core.ext.linker.GeneratedResource; import com.google.gwt.dev.util.DiskCache; -import com.google.gwt.dev.util.Util; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.ObjectInputStream; @@ -45,7 +45,12 @@ String partialPath, File file) { super(StandardLinkerContext.class, generatorType, partialPath); this.lastModified = file.lastModified(); - this.token = diskCache.writeByteArray(Util.readFileAsBytes(file)); + try { + this.token = diskCache.transferFromStream(new FileInputStream(file)); + } catch (FileNotFoundException e) { + throw new RuntimeException("Unable to open file '" + + file.getAbsolutePath() + "'", e); + } } @Override @@ -62,19 +67,17 @@ @Override public void writeTo(TreeLogger logger, OutputStream out) throws UnableToCompleteException { - diskCache.writeTo(token, out); + diskCache.transferToStream(token, out); } private void readObject(ObjectInputStream stream) throws IOException, ClassNotFoundException { stream.defaultReadObject(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - Util.copyNoClose(stream, baos); - token = diskCache.writeByteArray(baos.toByteArray()); + token = diskCache.transferFromStream(stream); } private void writeObject(ObjectOutputStream stream) throws IOException { stream.defaultWriteObject(); - diskCache.writeTo(token, stream); + diskCache.transferToStream(token, stream); } } ======================================= --- /trunk/dev/core/src/com/google/gwt/dev/jjs/UnifiedAst.java Tue Oct 27 19:12:04 2009 +++ /trunk/dev/core/src/com/google/gwt/dev/jjs/UnifiedAst.java Wed Oct 28 09:26:56 2009 @@ -20,10 +20,9 @@ import com.google.gwt.dev.Permutation; import com.google.gwt.dev.jjs.ast.JProgram; import com.google.gwt.dev.js.ast.JsProgram; -import com.google.gwt.dev.util.PerfLogger; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; +import com.google.gwt.dev.util.DiskCache; +import com.google.gwt.dev.util.Util; + import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; @@ -42,9 +41,9 @@ /** * Encapsulates the combined programs. */ - static final class AST { - private JProgram jProgram; - private JsProgram jsProgram; + static final class AST implements Serializable { + private final JProgram jProgram; + private final JsProgram jsProgram; public AST(JProgram jProgram, JsProgram jsProgram) { this.jProgram = jProgram; @@ -60,42 +59,7 @@ } } - private static AST deserializeAst(byte[] serializedAst) { - try { - PerfLogger.start("deserialize"); - ByteArrayInputStream bais = new ByteArrayInputStream(serializedAst); - ObjectInputStream is; - is = new ObjectInputStream(bais); - JProgram jprogram = (JProgram) is.readObject(); - JsProgram jsProgram = (JsProgram) is.readObject(); - return new AST(jprogram, jsProgram); - } catch (IOException e) { - throw new RuntimeException( - "Should be impossible for memory based streams", e); - } catch (ClassNotFoundException e) { - throw new RuntimeException( - "Should be impossible when deserializing in process", e); - } finally { - PerfLogger.end(); - } - } - - private static byte[] serializeAst(AST ast) { - try { - PerfLogger.start("serialize"); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ObjectOutputStream os = new ObjectOutputStream(baos); - os.writeObject(ast.getJProgram()); - os.writeObject(ast.getJsProgram()); - os.close(); - return baos.toByteArray(); - } catch (IOException e) { - throw new RuntimeException( - "Should be impossible for memory based streams", e); - } finally { - PerfLogger.end(); - } - } + private static final DiskCache diskCache = new DiskCache(); /** * The original AST; nulled out once consumed (by the first call to @@ -121,7 +85,7 @@ /** * The serialized AST. */ - private byte[] serializedAst; + private transient long serializedAstToken; public UnifiedAst(JJSOptions options, AST initialAst, boolean singlePermutation, Set<String> rebindRequests) { @@ -129,7 +93,8 @@ this.initialAst = initialAst; this.rebindRequests = Collections.unmodifiableSortedSet(new TreeSet<String>( rebindRequests)); - this.serializedAst = singlePermutation ? null : serializeAst(initialAst); + this.serializedAstToken = singlePermutation ? -1 + : diskCache.writeObject(initialAst); } /** @@ -140,7 +105,7 @@ this.initialAst = other.initialAst; other.initialAst = null; // steal its copy this.rebindRequests = other.rebindRequests; - this.serializedAst = other.serializedAst; + this.serializedAstToken = other.serializedAstToken; } /** @@ -179,7 +144,7 @@ public void prepare() { synchronized (myLockObject) { if (initialAst == null) { - initialAst = deserializeAst(serializedAst); + initialAst = diskCache.readObject(serializedAstToken, AST.class); } } } @@ -191,36 +156,39 @@ initialAst = null; return result; } else { - if (serializedAst == null) { + if (serializedAstToken < 0) { throw new IllegalStateException( "No serialized AST was cached and AST was already consumed."); } - return deserializeAst(serializedAst); + return diskCache.readObject(serializedAstToken, AST.class); } } } /** - * Re-initialize lock object. + * Re-initialize lock object; copy serialized AST straight to cache. */ - private Object readResolve() { + private void readObject(ObjectInputStream stream) throws IOException, + ClassNotFoundException { + stream.defaultReadObject(); myLockObject = new Object(); - return this; + serializedAstToken = diskCache.transferFromStream(stream); } /** * Force byte serialization of AST before writing. */ - private Object writeReplace() { - if (serializedAst == null) { - synchronized (myLockObject) { - if (initialAst == null) { - throw new IllegalStateException( - "No serialized AST was cached and AST was already consumed."); - } - serializedAst = serializeAst(initialAst); - } - } - return this; + private void writeObject(ObjectOutputStream stream) throws IOException { + stream.defaultWriteObject(); + if (serializedAstToken >= 0) { + // Copy the bytes. + diskCache.transferToStream(serializedAstToken, stream); + } else if (initialAst != null) { + // Serialize into raw bytes. + Util.writeObjectToStream(stream, stream); + } else { + throw new IllegalStateException( + "No serialized AST was cached and AST was already consumed."); + } } } ======================================= --- /trunk/dev/core/src/com/google/gwt/dev/util/DiskCache.java Wed Oct 28 09:10:53 2009 +++ /trunk/dev/core/src/com/google/gwt/dev/util/DiskCache.java Wed Oct 28 09:26:56 2009 @@ -19,6 +19,7 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.io.RandomAccessFile; import java.lang.ref.WeakReference; @@ -125,49 +126,44 @@ } /** - * Write a byte array to disk. + * Write the rest of the data in an input stream to disk. * * @return a handle to retrieve it later */ - public synchronized long writeByteArray(byte[] bytes) { + public synchronized long transferFromStream(InputStream in) { + byte[] buf = Util.takeThreadLocalBuf(); try { - if (!atEnd) { - file.seek(file.length()); - } - long position = file.getFilePointer(); - file.writeInt(bytes.length); - file.write(bytes); + long position = moveToEndPosition(); + + // Placeholder, we don't know the length yet. + file.writeInt(-1); + + // Transfer all the bytes. + int length = 0; + int bytesRead; + while ((bytesRead = in.read(buf)) != -1) { + file.write(buf, 0, bytesRead); + length += bytesRead; + } + + // Now go back and fill in the length. + file.seek(position); + file.writeInt(length); + // Don't eagerly seek the end, the next operation might be a read. + atEnd = false; return position; } catch (IOException e) { - throw new RuntimeException("Unable to write to byte cache", e); + throw new RuntimeException("Unable to read from byte cache", e); + } finally { + Util.releaseThreadLocalBuf(buf); } } - - public long writeObject(Object object) { - try { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - Util.writeObjectToStream(out, object); - return writeByteArray(out.toByteArray()); - } catch (IOException e) { - throw new RuntimeException("Unexpected IOException on in-memory stream", - e); - } - } - - /** - * Write a String to disk. - * - * @return a handle to retrieve it later - */ - public long writeString(String str) { - return writeByteArray(Util.getBytes(str)); - } /** * Reads bytes of data back from disk and writes them into the specified * output stream. */ - public synchronized void writeTo(long token, OutputStream out) { + public synchronized void transferToStream(long token, OutputStream out) { byte[] buf = Util.takeThreadLocalBuf(); try { atEnd = false; @@ -190,6 +186,42 @@ Util.releaseThreadLocalBuf(buf); } } + + /** + * Write a byte array to disk. + * + * @return a handle to retrieve it later + */ + public synchronized long writeByteArray(byte[] bytes) { + try { + long position = moveToEndPosition(); + file.writeInt(bytes.length); + file.write(bytes); + return position; + } catch (IOException e) { + throw new RuntimeException("Unable to write to byte cache", e); + } + } + + public long writeObject(Object object) { + try { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + Util.writeObjectToStream(out, object); + return writeByteArray(out.toByteArray()); + } catch (IOException e) { + throw new RuntimeException("Unexpected IOException on in-memory stream", + e); + } + } + + /** + * Write a String to disk. + * + * @return a handle to retrieve it later + */ + public long writeString(String str) { + return writeByteArray(Util.getBytes(str)); + } @Override protected synchronized void finalize() throws Throwable { @@ -203,4 +235,23 @@ file = null; } } -} + + /** + * Moves to the end of the file if necessary and returns the offset position. + * Caller must synchronize. + * + * @return the offset position of the end of the file + * @throws IOException + */ + private long moveToEndPosition() throws IOException { + // Get an end pointer. + if (atEnd) { + return file.getFilePointer(); + } else { + long position = file.length(); + file.seek(position); + atEnd = true; + return position; + } + } +} --~--~---------~--~----~------------~-------~--~----~ http://groups.google.com/group/Google-Web-Toolkit-Contributors -~----------~----~----~----~------~----~------~--~---
