JENA-1616: Better progress output. Project: http://git-wip-us.apache.org/repos/asf/jena/repo Commit: http://git-wip-us.apache.org/repos/asf/jena/commit/f5de3418 Tree: http://git-wip-us.apache.org/repos/asf/jena/tree/f5de3418 Diff: http://git-wip-us.apache.org/repos/asf/jena/diff/f5de3418
Branch: refs/heads/master Commit: f5de3418f8b619648e6cc863050db48d704ade91 Parents: 0dbc1c2 Author: Andy Seaborne <[email protected]> Authored: Tue Oct 9 16:53:37 2018 +0100 Committer: Andy Seaborne <[email protected]> Committed: Tue Oct 9 16:53:37 2018 +0100 ---------------------------------------------------------------------- .../java/org/apache/jena/atlas/lib/Timer.java | 18 ++-- jena-cmds/src/main/java/tdb2/tdbloader.java | 20 +++++ .../jena/tdb2/loader/base/LoaderBase.java | 54 ++++++++++-- .../apache/jena/tdb2/loader/base/LoaderOps.java | 9 +- .../jena/tdb2/loader/base/ProgressMonitor.java | 19 ++++- .../tdb2/loader/base/ProgressMonitorBasic.java | 48 +++++++++-- .../loader/base/ProgressMonitorContext.java | 61 ++++++++++++++ .../tdb2/loader/base/ProgressMonitorOutput.java | 88 +++++++++++++++----- .../tdb2/loader/base/ProgressStreamRDF.java | 16 ---- .../jena/tdb2/loader/basic/LoaderBasic.java | 6 +- .../jena/tdb2/loader/main/LoaderMain.java | 12 +-- .../sequential/BuilderSecondaryIndexes.java | 16 ++-- .../loader/sequential/LoaderSequential.java | 8 +- 13 files changed, 290 insertions(+), 85 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/jena/blob/f5de3418/jena-base/src/main/java/org/apache/jena/atlas/lib/Timer.java ---------------------------------------------------------------------- diff --git a/jena-base/src/main/java/org/apache/jena/atlas/lib/Timer.java b/jena-base/src/main/java/org/apache/jena/atlas/lib/Timer.java index 340ba37..c3e1ee4 100644 --- a/jena-base/src/main/java/org/apache/jena/atlas/lib/Timer.java +++ b/jena-base/src/main/java/org/apache/jena/atlas/lib/Timer.java @@ -32,13 +32,12 @@ public class Timer { public void startTimer() { if ( inTimer ) throw new AtlasException("Already in timer") ; - timeStart = System.currentTimeMillis() ; timeFinish = -1 ; inTimer = true ; } - /** Return time in milliseconds */ + /** Stop timing and return the elapsed time in milliseconds */ public long endTimer() { if ( !inTimer ) throw new AtlasException("Not in timer") ; @@ -47,21 +46,30 @@ public class Timer { return getTimeInterval() ; } + /** Read the timer - either the instantanious value (if running) or elapsed time (if finished). */ + public long read() { + return inTimer + ? System.currentTimeMillis() - timeStart + : timeFinish - timeStart; + } + + /** Read a running timer */ public long readTimer() { if ( !inTimer ) throw new AtlasException("Not in timer") ; - return System.currentTimeMillis() - timeStart ; + return read() ; } + /** Read an elapsed timer */ public long getTimeInterval() { if ( inTimer ) throw new AtlasException("Still timing") ; if ( timeFinish == -1 ) throw new AtlasException("No valid interval") ; - - return timeFinish - timeStart ; + return read() ; } + /** Helper function to format milliseconds as "%.3f" seconds */ static public String timeStr(long timeInterval) { return String.format("%.3f", timeInterval / 1000.0) ; } http://git-wip-us.apache.org/repos/asf/jena/blob/f5de3418/jena-cmds/src/main/java/tdb2/tdbloader.java ---------------------------------------------------------------------- diff --git a/jena-cmds/src/main/java/tdb2/tdbloader.java b/jena-cmds/src/main/java/tdb2/tdbloader.java index 1ee896e..2473a8f 100644 --- a/jena-cmds/src/main/java/tdb2/tdbloader.java +++ b/jena-cmds/src/main/java/tdb2/tdbloader.java @@ -18,12 +18,16 @@ package tdb2; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.List; import java.util.Objects; import jena.cmd.ArgDecl; import jena.cmd.CmdException; import org.apache.jena.atlas.lib.InternalErrorException; +import org.apache.jena.atlas.lib.ListUtils; import org.apache.jena.atlas.lib.Timer; import org.apache.jena.graph.Node; import org.apache.jena.graph.NodeFactory; @@ -109,6 +113,8 @@ public class tdbloader extends CmdTDBGraph { List<String> urls = getPositional(); if ( urls.size() == 0 ) urls.add("-"); + else + checkFiles(urls); if ( graphName == null ) { loadQuads(urls); @@ -128,6 +134,20 @@ public class tdbloader extends CmdTDBGraph { loadTriples(graphName, urls); } + // Check files exists before starting. + private void checkFiles(List<String> urls) { + List<String> problemFiles = + ListUtils.toList( + urls.stream() + .map(Paths::get) + .filter(p-> !Files.exists(p) || !Files.isRegularFile(p /*follow links*/) || !Files.isReadable(p) ) + .map(Path::toString) + ); + if ( ! problemFiles.isEmpty() ) { + throw new CmdException("Can't read files : ["+problemFiles+"]"); + } + } + private void loadTriples(String graphName, List<String> urls) { execBulkLoad(super.getDatasetGraph(), graphName, urls, showProgress); } http://git-wip-us.apache.org/repos/asf/jena/blob/f5de3418/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/LoaderBase.java ---------------------------------------------------------------------- diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/LoaderBase.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/LoaderBase.java index ce205e0..544310e 100644 --- a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/LoaderBase.java +++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/LoaderBase.java @@ -26,13 +26,13 @@ import org.apache.jena.query.TxnType; import org.apache.jena.sparql.core.DatasetGraph; import org.apache.jena.tdb2.loader.DataLoader; -/** Simple bulk loader framework. +/** + * Bulk loader framework. * <p> * It puts a write-transaction around the whole process if {@link #bulkUseTransaction} - * returns true and then calls abstract {@link #loadOne(String)} - * for each file. + * returns true. It calls {@link #loadOne} for each file. * <p> - * If a graph name is provided, it converts triples to quads in that named graph. + * If a graph name is provided, it converts triples to quads in that named graph. */ public abstract class LoaderBase implements DataLoader { @@ -62,7 +62,7 @@ public abstract class LoaderBase implements DataLoader { dsg.end(); } long totalElapsed = timer.endTimer(); - outputTime(totalElapsed); + outputSummary(totalElapsed); } @Override @@ -75,21 +75,57 @@ public abstract class LoaderBase implements DataLoader { @Override public void load(List<String> filenames) { - // Default implementation. + if ( filenames.isEmpty() ) { + output.print("No files to load"); + return; + } + + ProgressMonitor monitor = createProgressMonitor(output); + + boolean multipleFiles = (filenames.size()>1); + + String file1 = null; + if ( ! multipleFiles ) { + file1 = filenames.get(0); + if ( file1.equals("-") ) + file1 = "stdin"; + } try { - filenames.forEach(fn->loadOne(fn)); + if ( multipleFiles ) + monitor.startMessage("Start: "+filenames.size()+" files"); + else + monitor.startMessage("Start: "+file1); + monitor.start(); + filenames.forEach(fn->{ + if ( multipleFiles ) + monitor.startSection(); + loadOne(fn, monitor); + if ( multipleFiles ) + monitor.finishSection(); + }); + monitor.finish(); + if ( multipleFiles ) + monitor.finishMessage("Finished: "+filenames.size()+" files"); + else + monitor.finishMessage("Finished: "+file1); } catch (Exception ex) { finishException(ex); throw ex; } } + protected abstract ProgressMonitor createProgressMonitor(MonitorOutput output); + /** Subclasses must provide a setting. */ protected abstract boolean bulkUseTransaction(); - protected abstract void loadOne(String filename); + protected void loadOne(String filename, ProgressMonitor monitor) { + String label = LoaderOps.label(filename); + monitor.setLabel(label); + LoaderOps.inputFile(stream(), filename, monitor); + } - protected void outputTime(long totalElapsed) { + protected void outputSummary(long totalElapsed) { if ( output != null ) { long count = countTriples()+countQuads(); String label = "Triples/Quads"; http://git-wip-us.apache.org/repos/asf/jena/blob/f5de3418/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/LoaderOps.java ---------------------------------------------------------------------- diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/LoaderOps.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/LoaderOps.java index 88b95b7..89e8941 100644 --- a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/LoaderOps.java +++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/LoaderOps.java @@ -79,7 +79,7 @@ public class LoaderOps { } /** Calculate a label for a progress monitor. */ - private static String label(String fileName) { + public static String label(String fileName) { String basename = FileOps.splitDirFile(fileName).get(1); return basename; } @@ -100,17 +100,16 @@ public class LoaderOps { * "no output". */ public static void inputFile(StreamRDF sink, String source, ProgressMonitor monitor) { - if ( monitor != null ) { sink = new ProgressStreamRDF(sink, monitor); - monitor.start(); + //monitor.start(); } sink.start(); RDFDataMgr.parse(sink, source); sink.finish(); if ( monitor != null ) { - monitor.finish(); - monitor.finishMessage("Data: " + source); + //monitor.finish(); + //monitor.finishMessage("Data"); } } http://git-wip-us.apache.org/repos/asf/jena/blob/f5de3418/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitor.java ---------------------------------------------------------------------- diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitor.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitor.java index 7f114b5..9c3934b 100644 --- a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitor.java +++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitor.java @@ -20,25 +20,31 @@ package org.apache.jena.tdb2.loader.base; /** Interface {@code ProgressMonitor} - monitor progress.*/ public interface ProgressMonitor { - public void startMessage(); - /** Output the starting message. * The format is implementation dependent. */ public void startMessage(String message); - + /** * Output the finishing message. * The format is implementation dependent. */ public void finishMessage(String message); + public void setLabel(String label); + /** Start and start timing. This should be paired with a call to {@link #finish()}. */ public void start(); + + /** Start a section within the overall start-finish. */ + public void startSection(); + /** Finish a section within the overall start-finish. */ + public void finishSection(); + /** * Finish and stop timing. The total time is available with {@link #getTime} and the - * numbe rof items processes with {@link #getTicks()}. + * number of items processes with {@link #getTicks()}. */ public void finish(); @@ -51,4 +57,9 @@ public interface ProgressMonitor { /** Return the elapsed time taken - this is only valid after {@link #finish()} has been called. */ public long getTime(); + /** Return the number of ticks. Valid after {@link #startSection()} has been called. */ + public long getSectionTicks(); + + /** Return the elapsed section time taken. */ + public long getSectionTime(); } http://git-wip-us.apache.org/repos/asf/jena/blob/f5de3418/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorBasic.java ---------------------------------------------------------------------- diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorBasic.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorBasic.java index 81ee9a4..19ccc2b 100644 --- a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorBasic.java +++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorBasic.java @@ -19,20 +19,24 @@ package org.apache.jena.tdb2.loader.base; import org.apache.jena.atlas.lib.Timer; +import org.apache.jena.tdb2.TDBException; /** Simple {@link ProgressMonitor} that records time and ticks but does not print anything */ public class ProgressMonitorBasic implements ProgressMonitor { - + // Overall private Timer timer = new Timer(); private long timeInMillis = -1; private long tickCounter = 0; + //Section + private boolean inSection = false; + private int sectionCounter = 0; + private Timer sectionTimer = null; + private long sectionTimeInMillis = -1; + private long sectionTickCounter = 0; public ProgressMonitorBasic() {} @Override - public void startMessage() {} - - @Override public void startMessage(String message) {} @Override @@ -49,7 +53,30 @@ public class ProgressMonitorBasic implements ProgressMonitor { } @Override - public void tick() { tickCounter++; } + public void startSection() { + if ( inSection ) + throw new TDBException("startSection: Already in section"); + inSection = true; + sectionCounter++; + sectionTimer = new Timer(); + sectionTimeInMillis = 0; + sectionTickCounter = 0; + } + + @Override + public void finishSection() { + if ( ! inSection ) + throw new TDBException("finishSection: Not in section"); + inSection = false; + sectionTimeInMillis = sectionTimer.endTimer(); + } + + @Override + public void tick() { + tickCounter++; + if ( inSection ) + sectionTickCounter++; + } @Override public long getTicks() { @@ -61,4 +88,15 @@ public class ProgressMonitorBasic implements ProgressMonitor { return timeInMillis; } + @Override + public long getSectionTicks() { + return sectionTickCounter; + } + @Override + public long getSectionTime() { + return sectionTimeInMillis; + } + + @Override + public void setLabel(String label) {} } http://git-wip-us.apache.org/repos/asf/jena/blob/f5de3418/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorContext.java ---------------------------------------------------------------------- diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorContext.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorContext.java new file mode 100644 index 0000000..14d9811 --- /dev/null +++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorContext.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.tdb2.loader.base; + +import org.apache.jena.atlas.lib.Timer; +import org.apache.jena.atlas.logging.Log; + +/** + * The counting state of a {@link ProgressMonitor}. This can be used to across different + * {@code ProgressMonitor}s (sequentially) to give running totals + */ + +public class ProgressMonitorContext { + /*package*/ long ticks; + /*package*/ Timer timer; + private int depth = 0; + + public ProgressMonitorContext(long ticks, Timer timer) { + super(); + this.ticks = ticks; + this.timer = timer; + } + + public void tick() { ticks++; } + + public void start() { + if ( depth == 0 ) + timer.startTimer(); + depth++; + } + public void finish() { + --depth; + if ( depth < 0 ) { + Log.error(this, "Misaligned start/finish"); + return; + } + if ( depth == 0 ) + timer.endTimer(); + } + + //public int getDepth() { return depth; } + + public long getElapsed() { return depth < 0 ? -1 : timer.readTimer(); } + +} http://git-wip-us.apache.org/repos/asf/jena/blob/f5de3418/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorOutput.java ---------------------------------------------------------------------- diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorOutput.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorOutput.java index 9720069..552d2de 100644 --- a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorOutput.java +++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorOutput.java @@ -23,6 +23,7 @@ import static org.apache.jena.atlas.lib.DateTimeUtils.nowAsString ; import java.util.Objects; import org.apache.jena.atlas.lib.Timer; +import org.apache.jena.tdb2.TDBException; import org.slf4j.Logger ; /** Progress monitor - output lines to show the progress of some long running operation. @@ -34,9 +35,17 @@ public class ProgressMonitorOutput implements ProgressMonitor { private final long tickPoint; private final int superTick; private final Timer timer; - private Timer getTimer() { return timer; } + private Timer getTimer() { return timer; } + + //Section + private boolean inSection = false; + private int sectionCounter = 0; + private Timer sectionTimer = null; + private long sectionTimeInMillis = -1; + private long sectionTickCounter = 0; - private final String label; + // Current label. + private String label; // Counters - this monitor. private long counterBatch = 0; @@ -72,41 +81,38 @@ public class ProgressMonitorOutput implements ProgressMonitor { */ public ProgressMonitorOutput(String label, long tickPoint, int superTick, MonitorOutput output) { this.output = output; - this.label = label; + setLabel(label); this.tickPoint = tickPoint; this.superTick = superTick; this.timer = new Timer(); } - /** Print a start message using the label */ - @Override - public void startMessage() { - startMessage(null) ; - } +// /** Print a start message using the label */ +// @Override +// public void startMessage() { +// startMessage(null) ; +// } /** Print a start message using a different string. */ @Override public void startMessage(String msg) { if ( msg != null ) output.print(msg) ; - else - output.print("Start: "+label) ; } + //public void startSource(String msg) { + //public void finishSource(String msg) { + @Override public void finishMessage(String msg) { // Elapsed. - long timePoint = getTimer().getTimeInterval(); - long counterTotalMsg = getRunningTotal(); - - // /1000L is milli to second conversion + long timePoint = getTimer().read(); if ( timePoint != 0 ) { double time = timePoint / 1000.0; - long runAvgRate = (counterTotalMsg * 1000L) / timePoint; - - print("%s: %,d %s %.2fs (Avg: %,d)", msg, counterTotalMsg, label, time, runAvgRate); + long runAvgRate = (getRunningTotal() * 1000L) / timePoint; + print("%s: %,d tuples in %.2fs (Avg: %,d)", msg, getTicks(), time, runAvgRate); } else - print("%s: %,d %s (Avg: ----)", msg, counterTotalMsg, label); + print("%s: %,d (Avg: ----)", msg, getTicks()); } @Override @@ -118,14 +124,19 @@ public class ProgressMonitorOutput implements ProgressMonitor { @Override public void finish() { - timeTotalMillis = getTimer().endTimer(); + // XXX + getTimer().endTimer(); + timeTotalMillis = getTimer().getTimeInterval(); } @Override public void tick() { + // The ticking counterBatch++; counterTotal++; - + if ( inSection ) + sectionTickCounter++; + // Report overall progress if ( tickPoint(getRunningTotal(), tickPoint) ) { long timePoint = getTimer().readTimer(); long thisTime = timePoint - lastTime; @@ -133,6 +144,8 @@ public class ProgressMonitorOutput implements ProgressMonitor { // *1000L is milli to second conversion if ( thisTime != 0 && timePoint != 0 ) { long batchAvgRate = (counterBatch * 1000L) / thisTime; + // XXX Too large : first after file switch. ???timePoint is wrong. + //System.err.printf("** %d %d\n",getRunningTotal(), timePoint ); long runAvgRate = (getRunningTotal() * 1000L) / timePoint; print("Add: %,d %s (Batch: %,d / Avg: %,d)", getRunningTotal(), label, batchAvgRate, runAvgRate); } else { @@ -161,6 +174,36 @@ public class ProgressMonitorOutput implements ProgressMonitor { public long getTime() { return timeTotalMillis; } + + @Override + public void startSection() { + if ( inSection ) + throw new TDBException("startSection: Already in section"); + inSection = true; + sectionCounter++; + sectionTimer = new Timer(); + sectionTimer.startTimer(); + sectionTimeInMillis = 0; + sectionTickCounter = 0; + } + + @Override + public void finishSection() { + if ( ! inSection ) + throw new TDBException("finishSection: Not in section"); + print(" End file: %s (triples/quads = %,d)", label, sectionTickCounter); + inSection = false; + sectionTimeInMillis = sectionTimer.endTimer(); + } + + @Override + public long getSectionTicks() { + return sectionTickCounter; + } + @Override + public long getSectionTime() { + return sectionTimeInMillis; + } protected void elapsed(long timerReading) { float elapsedSecs = timerReading / 1000F; @@ -177,4 +220,9 @@ public class ProgressMonitorOutput implements ProgressMonitor { return counter % quantum == 0; } + @Override + public void setLabel(String label) { + this.label = label; + } + } http://git-wip-us.apache.org/repos/asf/jena/blob/f5de3418/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressStreamRDF.java ---------------------------------------------------------------------- diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressStreamRDF.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressStreamRDF.java index e6c7b59..39b31af 100644 --- a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressStreamRDF.java +++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressStreamRDF.java @@ -36,22 +36,6 @@ public class ProgressStreamRDF extends StreamRDFWrapper { this.monitor = monitor ; } - // Better that the app call start/finish on the monitor so that a number of - // inputs on the stream can call start/finish. i.e the monitor can be used - // for a batch of oeprations. - -// @Override -// public void start() { -// monitor.start(); -// super.start(); -// } -// -// @Override -// public void finish() { -// super.finish(); -// monitor.finish(); -// } - @Override public void triple(Triple triple) { super.triple(triple); http://git-wip-us.apache.org/repos/asf/jena/blob/f5de3418/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/basic/LoaderBasic.java ---------------------------------------------------------------------- diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/basic/LoaderBasic.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/basic/LoaderBasic.java index e30e0f5..41dbbbf 100644 --- a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/basic/LoaderBasic.java +++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/basic/LoaderBasic.java @@ -27,6 +27,8 @@ import org.apache.jena.sparql.core.DatasetGraph; import org.apache.jena.tdb2.loader.base.LoaderBase; import org.apache.jena.tdb2.loader.base.LoaderOps; import org.apache.jena.tdb2.loader.base.MonitorOutput; +import org.apache.jena.tdb2.loader.base.ProgressMonitor; +import org.apache.jena.tdb2.loader.base.ProgressMonitorOutput; /** Simple bulk loader. Algorithm: Parser to dataset. */ public class LoaderBasic extends LoaderBase { @@ -53,8 +55,8 @@ public class LoaderBasic extends LoaderBase { } @Override - protected void loadOne(String source) { - LoaderOps.inputFile(dest, source, output, DataTickPoint, DataSuperTick); + protected ProgressMonitor createProgressMonitor(MonitorOutput output) { + return ProgressMonitorOutput.create(output, "<unset>", DataTickPoint, DataSuperTick); } @Override http://git-wip-us.apache.org/repos/asf/jena/blob/f5de3418/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/main/LoaderMain.java ---------------------------------------------------------------------- diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/main/LoaderMain.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/main/LoaderMain.java index 8bd158d..8b0fdf9 100644 --- a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/main/LoaderMain.java +++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/main/LoaderMain.java @@ -96,16 +96,16 @@ public class LoaderMain extends LoaderBase implements DataLoader { public static final int DataSuperTick = 10; public static final int IndexTickPoint = 1_000_000; public static final int IndexSuperTick = 10; - + private final LoaderPlan loaderPlan; - + private final DatasetGraphTDB dsgtdb; private final StreamRDF stream; private final Map<String, TupleIndex> indexMap; private final StreamRDFCounting dataInput; private final List<BulkStartFinish> dataProcess = new ArrayList<>(); - + public LoaderMain(LoaderPlan loaderPlan, DatasetGraph dsg, MonitorOutput output) { this(loaderPlan, dsg, null, output); } @@ -284,7 +284,7 @@ public class LoaderMain extends LoaderBase implements DataLoader { // Add to processes - we can wait later if we do not touched indexes being built. processes.add(indexer); PhasedOps.ReplayResult result = PhasedOps.replay(srcIdx, dest, output); - // End read tranaction on srcIdx + // End read transaction on srcIdx transaction.end(); String timeStr = "---"; @@ -327,7 +327,7 @@ public class LoaderMain extends LoaderBase implements DataLoader { } @Override - protected void loadOne(String filename) { - LoaderOps.inputFile(stream, filename, output, DataTickPoint, DataSuperTick); + protected ProgressMonitor createProgressMonitor(MonitorOutput output) { + return ProgressMonitorOutput.create(output, "<unset>", DataTickPoint, DataSuperTick); } } http://git-wip-us.apache.org/repos/asf/jena/blob/f5de3418/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/BuilderSecondaryIndexes.java ---------------------------------------------------------------------- diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/BuilderSecondaryIndexes.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/BuilderSecondaryIndexes.java index b1bec81..8484a39 100644 --- a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/BuilderSecondaryIndexes.java +++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/BuilderSecondaryIndexes.java @@ -18,7 +18,6 @@ package org.apache.jena.tdb2.loader.sequential; -import org.apache.jena.atlas.lib.Timer; import org.apache.jena.tdb2.loader.base.LoaderOps; import org.apache.jena.tdb2.loader.base.MonitorOutput; import org.apache.jena.tdb2.loader.base.ProgressMonitor; @@ -26,27 +25,24 @@ import org.apache.jena.tdb2.loader.base.ProgressMonitorOutput; import org.apache.jena.tdb2.store.tupletable.TupleIndex; /** - * This interface is the mechanism for building indexes given that at leasts one index + * This interface is the mechanism for building indexes given that at least one index * already exists (the "primary", which normally is SPO or GSPO). */ public class BuilderSecondaryIndexes { - public static void createSecondaryIndexes(MonitorOutput output, TupleIndex primaryIndex, TupleIndex[] secondaryIndexes) - { - Timer timer = new Timer() ; - timer.startTimer() ; + public static void createSecondaryIndexes(MonitorOutput output, TupleIndex primaryIndex, TupleIndex[] secondaryIndexes) { boolean printTiming = true; for ( TupleIndex index : secondaryIndexes ) { + String msg = primaryIndex.getName()+"->"+index.getName(); if ( index != null ) { - ProgressMonitor monitor = ProgressMonitorOutput.create(output, index.getName(), + ProgressMonitor monitor = ProgressMonitorOutput.create(output, msg, LoaderSequential.IndexTickPoint, LoaderSequential.IndexSuperTick); - monitor.startMessage(); + monitor.startMessage(msg); monitor.start(); - long time1 = timer.readTimer() ; LoaderOps.copyIndex(primaryIndex.all(), new TupleIndex[]{index}, monitor) ; - long time2 = timer.readTimer() ; + monitor.finish(); monitor.finishMessage(index.getName()+" indexing: "); } http://git-wip-us.apache.org/repos/asf/jena/blob/f5de3418/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/LoaderSequential.java ---------------------------------------------------------------------- diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/LoaderSequential.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/LoaderSequential.java index 32c2244..98d98dd 100644 --- a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/LoaderSequential.java +++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/LoaderSequential.java @@ -29,6 +29,8 @@ import org.apache.jena.tdb2.loader.BulkLoaderException; import org.apache.jena.tdb2.loader.base.LoaderBase; import org.apache.jena.tdb2.loader.base.LoaderOps; import org.apache.jena.tdb2.loader.base.MonitorOutput; +import org.apache.jena.tdb2.loader.base.ProgressMonitor; +import org.apache.jena.tdb2.loader.base.ProgressMonitorOutput; import org.apache.jena.tdb2.store.DatasetGraphTDB; import org.apache.jena.tdb2.sys.TDBInternal; @@ -103,10 +105,10 @@ public class LoaderSequential extends LoaderBase { } @Override - protected void loadOne(String filename) { - LoaderOps.inputFile(stream, filename, output, DataTickPoint, DataSuperTick); + protected ProgressMonitor createProgressMonitor(MonitorOutput output) { + return ProgressMonitorOutput.create(output, "<unset>", DataTickPoint, DataSuperTick); } - + @Override public boolean bulkUseTransaction() { return true;
