This is an automated email from the ASF dual-hosted git repository. spmallette pushed a commit to branch tinkergraph-storage in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit e252a234c133bc86f49af0fb554d1cf8c04304ea Author: Stephen Mallette <[email protected]> AuthorDate: Tue Aug 4 15:45:06 2026 +0000 Auto-compact TinkerStorageGraph log to bound its growth A long-running graph that is never explicitly closed grew its append log without bound, and its restart replay cost grew with it. The GraphBinary engine now tracks appended log bytes and folds the log into a snapshot on commit once it exceeds a threshold, configurable via gremlin.tinkergraph.storage.compactThreshold (default 64MB, 0 to disable). A new no-op-by-default TinkerStorage.maybeCompact SPI hook drives this, so existing engines are unaffected. Compaction runs inline under the commit lock. Assisted-by: Claude Code:claude-opus-4-8 --- CHANGELOG.asciidoc | 2 +- .../gremlin/tinkergraph/structure/TinkerGraph.java | 8 ++++ .../tinkergraph/structure/TinkerTransaction.java | 3 ++ .../structure/storage/GraphBinaryStorage.java | 23 ++++++++++ .../structure/storage/TinkerStorage.java | 12 +++++ .../structure/storage/GraphBinaryStorageTest.java | 51 ++++++++++++++++++++++ 6 files changed, 98 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 40c46500c6..2fafc49051 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -28,7 +28,7 @@ image::https://raw.githubusercontent.com/apache/tinkerpop/master/docs/static/ima * Fixed `gremlin-go` to report a malformed or truncated GraphBinary response as a deserialization error rather than a bare decoder message. * Made `TinkerGraph` an interface and renamed the in-memory implementation to `TinkerMemoryGraph`; `TinkerGraph.open()` and `gremlin.graph=...TinkerGraph` behave as before. *(breaking)* * Renamed `TinkerTransactionGraph` to `TinkerStorageGraph`. *(breaking)* -* Added a pluggable storage layer to `TinkerStorageGraph` that durably persists each committed transaction to disk, selected with the `gremlin.tinkergraph.storage` config key and shipping a GraphBinary engine, with a `gremlin.tinkergraph.storage.sync` key to choose `commit` (fsync per commit) or `os` durability; a storage location is locked to a single writer, so opening one already in use fails fast. +* Added a pluggable storage layer to `TinkerStorageGraph` that durably persists each committed transaction to disk, selected with the `gremlin.tinkergraph.storage` config key and shipping a GraphBinary engine, with a `gremlin.tinkergraph.storage.sync` key to choose `commit` (fsync per commit) or `os` durability; a storage location is locked to a single writer, so opening one already in use fails fast; the append log auto-compacts once it exceeds `gremlin.tinkergraph.storage.compactThresh [...] * Removed automatic persistence from `TinkerMemoryGraph`, which is now purely in-memory and ignores `gremlin.tinkergraph.graphLocation`/`graphFormat`. Use `TinkerStorageGraph` for durability or `g.io()` for interchange. *(breaking)* [[release-4-0-0-beta-3]] diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java index 3bb5200bef..65a02537c6 100644 --- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java +++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java @@ -67,6 +67,14 @@ public interface TinkerGraph extends Graph { * {@code org.apache.tinkerpop.gremlin.tinkergraph.structure.storage.SyncMode}. */ String GREMLIN_TINKERGRAPH_STORAGE_SYNC = "gremlin.tinkergraph.storage.sync"; + /** + * The size in bytes at which a {@link TinkerStorageGraph} storage engine automatically compacts its append log on + * commit, bounding the log growth (and restart replay cost) of a long-running graph that is never explicitly + * closed. Defaults to 67108864 (64 MB). Set to {@code 0} to disable automatic compaction and rely on + * {@code close()} or an explicit {@code compact()}. Only meaningful when {@link #GREMLIN_TINKERGRAPH_STORAGE} is + * set. + */ + String GREMLIN_TINKERGRAPH_STORAGE_COMPACT_THRESHOLD = "gremlin.tinkergraph.storage.compactThreshold"; String GREMLIN_TINKERGRAPH_ALLOW_NULL_PROPERTY_VALUES = "gremlin.tinkergraph.allowNullPropertyValues"; String GREMLIN_TINKERGRAPH_SERVICE = "gremlin.tinkergraph.service"; String GREMLIN_TINKERGRAPH_VERTEX_LABEL_CARDINALITY = "gremlin.tinkergraph.vertexLabelCardinality"; diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerTransaction.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerTransaction.java index 50c48bde80..36d5167c13 100644 --- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerTransaction.java +++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerTransaction.java @@ -186,6 +186,9 @@ final class TinkerTransaction extends AbstractThreadLocalTransaction { try { graph.storage.persist(txVersion, toVertexMutations(changedVertices), toEdgeMutations(changedEdges)); graph.storage.flush(); + // bound log growth for a long-running graph that is never explicitly closed; no-op unless the + // engine's accumulated log has crossed its threshold + graph.storage.maybeCompact(graph); } finally { graph.storageCommitLock.unlock(); } diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java index 9b31349a2a..c9eecba9e5 100644 --- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java +++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java @@ -91,9 +91,16 @@ public final class GraphBinaryStorage implements TinkerStorage { private File snapshotFile; private File logFile; + /** + * Default automatic-compaction threshold: 64 MB of appended log since the last compaction. + */ + static final long DEFAULT_COMPACT_THRESHOLD_BYTES = 64L * 1024 * 1024; + private DataOutputStream logOut; private FileOutputStream logFos; private SyncMode syncMode = SyncMode.COMMIT; + private long compactThresholdBytes = DEFAULT_COMPACT_THRESHOLD_BYTES; + private long logBytesSinceCompaction = 0; private boolean closed = false; @Override @@ -106,6 +113,10 @@ public final class GraphBinaryStorage implements TinkerStorage { this.snapshotFile = new File(directory, SNAPSHOT_FILE); this.logFile = new File(directory, LOG_FILE); this.syncMode = SyncMode.fromConfigValue(config.getString(TinkerGraph.GREMLIN_TINKERGRAPH_STORAGE_SYNC, null)); + this.compactThresholdBytes = config.getLong( + TinkerGraph.GREMLIN_TINKERGRAPH_STORAGE_COMPACT_THRESHOLD, DEFAULT_COMPACT_THRESHOLD_BYTES); + // seed the counter with any pre-existing log so a graph reopened with a large log still compacts promptly + this.logBytesSinceCompaction = logFile.exists() ? logFile.length() : 0; ensureDirectory(); } @@ -210,6 +221,7 @@ public final class GraphBinaryStorage implements TinkerStorage { try { final byte[] frame = encodeRecord(txVersion, changedVertices, changedEdges); writeFrame(logOut, frame); + logBytesSinceCompaction += Integer.BYTES + frame.length; // length prefix + payload } catch (IOException ex) { throw new UncheckedIOException("Could not append transaction to storage log", ex); } @@ -304,6 +316,17 @@ public final class GraphBinaryStorage implements TinkerStorage { } catch (IOException ex) { throw new UncheckedIOException("Could not finalize storage snapshot", ex); } + + // the log is now empty; the accumulated state lives in the snapshot + logBytesSinceCompaction = 0; + } + + @Override + public void maybeCompact(final AbstractTinkerGraph graph) { + if (closed || compactThresholdBytes <= 0) + return; + if (logBytesSinceCompaction >= compactThresholdBytes) + compact(graph); } /** diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/TinkerStorage.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/TinkerStorage.java index 49dcedb1ba..ab4745dcf2 100644 --- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/TinkerStorage.java +++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/TinkerStorage.java @@ -90,6 +90,18 @@ public interface TinkerStorage extends AutoCloseable { */ void compact(AbstractTinkerGraph graph); + /** + * Compact if the engine's accumulated data has grown past its own threshold, otherwise do nothing. Called on the + * commit path after {@link #flush()} (while the graph's commit lock is held) so a long-running graph that is never + * explicitly closed does not grow its backing storage without bound. The default is a no-op, leaving compaction + * entirely under the control of {@link #compact(AbstractTinkerGraph)} and {@link #close()}. + * + * @param graph the graph whose current committed state would be snapshotted + */ + default void maybeCompact(final AbstractTinkerGraph graph) { + // no-op by default; engines that accumulate an on-disk log override this to bound its growth + } + /** * {@inheritDoc} * <p/> diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java index 05e9970504..8113e03d18 100644 --- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java +++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java @@ -105,6 +105,57 @@ public class GraphBinaryStorageTest extends AbstractTinkerStorageConformanceTest graph.close(); } + @Test + public void shouldAutoCompactWhenLogExceedsThreshold() { + // a small threshold makes automatic compaction fire mid-run, without any explicit compact()/close() + final Configuration conf = config(); + conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_STORAGE_COMPACT_THRESHOLD, 2048L); + final String location = conf.getString(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION); + TinkerStorageGraph graph = TinkerStorageGraph.open(conf); + try { + for (int i = 0; i < 200; i++) { + graph.addVertex(T.id, i, "value", i, "pad", "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"); + graph.tx().commit(); + } + // auto-compaction should have folded the log into a snapshot and truncated it well below the total + // bytes written, so the live log stays bounded rather than growing with every commit + final File logFile = new File(location, GraphBinaryStorage.LOG_FILE); + final File snapshotFile = new File(location, GraphBinaryStorage.SNAPSHOT_FILE); + assertTrue("expected a snapshot from auto-compaction", snapshotFile.exists()); + assertTrue("expected the live log to stay bounded, was " + logFile.length(), + logFile.length() < 8192); + } finally { + graph.close(); + } + + // data must survive across reopen despite the mid-run compactions + graph = TinkerStorageGraph.open(conf); + try { + assertEquals(200, countOf(graph.vertices())); + assertEquals(Integer.valueOf(199), graph.vertices(199).next().value("value")); + } finally { + graph.close(); + } + } + + @Test + public void shouldNotAutoCompactWhenThresholdIsZero() { + final Configuration conf = config(); + conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_STORAGE_COMPACT_THRESHOLD, 0L); + final String location = conf.getString(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION); + final TinkerStorageGraph graph = TinkerStorageGraph.open(conf); + try { + for (int i = 0; i < 50; i++) { + graph.addVertex(T.id, i, "pad", "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"); + graph.tx().commit(); + } + // with auto-compaction disabled, no snapshot appears until close()/compact() + assertTrue(!new File(location, GraphBinaryStorage.SNAPSHOT_FILE).exists()); + } finally { + graph.close(); + } + } + @Test public void shouldRecoverFromTruncatedTrailingFrame() throws Exception { TinkerStorageGraph graph = open();
