Updated Branches: refs/heads/trunk 585511f7c -> 419445017
GIRAPH-834 Project: http://git-wip-us.apache.org/repos/asf/giraph/repo Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/41944501 Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/41944501 Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/41944501 Branch: refs/heads/trunk Commit: 419445017610d1c65db1bfafa0cd09e094b7156f Parents: 585511f Author: Claudio Martella <[email protected]> Authored: Fri Jan 31 14:21:54 2014 +0100 Committer: Claudio Martella <[email protected]> Committed: Fri Jan 31 14:21:54 2014 +0100 ---------------------------------------------------------------------- CHANGELOG | 2 ++ .../java/org/apache/giraph/bsp/CentralizedServiceWorker.java | 6 +++++- .../main/java/org/apache/giraph/graph/GraphTaskManager.java | 8 ++++++-- .../java/org/apache/giraph/metrics/AggregatedMetrics.java | 4 ++++ .../main/java/org/apache/giraph/worker/BspServiceWorker.java | 8 ++++++-- giraph-dist/pom.xml | 4 ---- 6 files changed, 23 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/giraph/blob/41944501/CHANGELOG ---------------------------------------------------------------------- diff --git a/CHANGELOG b/CHANGELOG index d88ed41..d2f5980 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,8 @@ Giraph Change Log Release 1.1.0 - unreleased + GIRAPH-834: Metrcis missing superstep time (armax00 via claudio) + GIRAPH-819: Number of Containers Required for a Job (Rafal Wojdyla via ereisman) GIRAPH-832: Test don't finish after GIRAPH-792 (majakabiljo) http://git-wip-us.apache.org/repos/asf/giraph/blob/41944501/giraph-core/src/main/java/org/apache/giraph/bsp/CentralizedServiceWorker.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/bsp/CentralizedServiceWorker.java b/giraph-core/src/main/java/org/apache/giraph/bsp/CentralizedServiceWorker.java index 29567c0..e5d0ae1 100644 --- a/giraph-core/src/main/java/org/apache/giraph/bsp/CentralizedServiceWorker.java +++ b/giraph-core/src/main/java/org/apache/giraph/bsp/CentralizedServiceWorker.java @@ -25,6 +25,7 @@ import org.apache.giraph.graph.GraphTaskManager; import org.apache.giraph.graph.VertexEdgeCount; import org.apache.giraph.io.superstep_output.SuperstepOutput; import org.apache.giraph.master.MasterInfo; +import org.apache.giraph.metrics.GiraphTimerContext; import org.apache.giraph.partition.PartitionOwner; import org.apache.giraph.partition.PartitionStats; import org.apache.giraph.partition.PartitionStore; @@ -129,10 +130,13 @@ public interface CentralizedServiceWorker<I extends WritableComparable, * worker level statistics after the computation. * * @param partitionStatsList All the partition stats for this worker + * @param superstepTimerContext superstep timer context only given when the + * function needs to stop the timer, otherwise null. * @return Stats of the superstep completion */ FinishedSuperstepStats finishSuperstep( - List<PartitionStats> partitionStatsList); + List<PartitionStats> partitionStatsList, + GiraphTimerContext superstepTimerContext); /** * Get the partition id that a vertex id would belong to. http://git-wip-us.apache.org/repos/asf/giraph/blob/41944501/giraph-core/src/main/java/org/apache/giraph/graph/GraphTaskManager.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/GraphTaskManager.java b/giraph-core/src/main/java/org/apache/giraph/graph/GraphTaskManager.java index 064c71f..573bd95 100644 --- a/giraph-core/src/main/java/org/apache/giraph/graph/GraphTaskManager.java +++ b/giraph-core/src/main/java/org/apache/giraph/graph/GraphTaskManager.java @@ -410,8 +410,12 @@ public class GraphTaskManager<I extends WritableComparable, V extends Writable, private FinishedSuperstepStats completeSuperstepAndCollectStats( List<PartitionStats> partitionStatsList, GiraphTimerContext superstepTimerContext) { - finishedSuperstepStats = serviceWorker.finishSuperstep(partitionStatsList); - superstepTimerContext.stop(); + + // the superstep timer is stopped inside the finishSuperstep function + // (otherwise metrics are not available at the end of the computation + // using giraph.metrics.enable=true). + finishedSuperstepStats = + serviceWorker.finishSuperstep(partitionStatsList, superstepTimerContext); if (conf.metricsEnabled()) { GiraphMetrics.get().perSuperstep().printSummary(System.err); } http://git-wip-us.apache.org/repos/asf/giraph/blob/41944501/giraph-core/src/main/java/org/apache/giraph/metrics/AggregatedMetrics.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/AggregatedMetrics.java b/giraph-core/src/main/java/org/apache/giraph/metrics/AggregatedMetrics.java index 9ce074b..83681e5 100644 --- a/giraph-core/src/main/java/org/apache/giraph/metrics/AggregatedMetrics.java +++ b/giraph-core/src/main/java/org/apache/giraph/metrics/AggregatedMetrics.java @@ -64,6 +64,8 @@ public class AggregatedMetrics { */ public AggregatedMetrics add(WorkerSuperstepMetrics workerMetrics, String hostname) { + add(GraphTaskManager.TIMER_SUPERSTEP_TIME, + workerMetrics.getSuperstepTimer(), hostname); add(GraphTaskManager.TIMER_COMMUNICATION_TIME, workerMetrics.getCommTimer(), hostname); add(GraphTaskManager.TIMER_COMPUTE_ALL, @@ -85,6 +87,7 @@ public class AggregatedMetrics { * @return this */ public AggregatedMetrics print(long superstep, PrintStream out) { + AggregatedMetric superstepTime = get(GraphTaskManager.TIMER_SUPERSTEP_TIME); AggregatedMetric commTime = get(GraphTaskManager.TIMER_COMMUNICATION_TIME); AggregatedMetric computeAll = get(GraphTaskManager.TIMER_COMPUTE_ALL); AggregatedMetric timeToFirstMsg = @@ -95,6 +98,7 @@ public class AggregatedMetrics { out.println(); out.println("--- METRICS: superstep " + superstep + " ---"); + printAggregatedMetric(out, "superstep time", "ms", superstepTime); printAggregatedMetric(out, "compute all partitions", "ms", computeAll); printAggregatedMetric(out, "user compute time", "ms", userComputeTime); printAggregatedMetric(out, "network communication time", "ms", commTime); http://git-wip-us.apache.org/repos/asf/giraph/blob/41944501/giraph-core/src/main/java/org/apache/giraph/worker/BspServiceWorker.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/worker/BspServiceWorker.java b/giraph-core/src/main/java/org/apache/giraph/worker/BspServiceWorker.java index 13de188..88a642a 100644 --- a/giraph-core/src/main/java/org/apache/giraph/worker/BspServiceWorker.java +++ b/giraph-core/src/main/java/org/apache/giraph/worker/BspServiceWorker.java @@ -587,7 +587,7 @@ public class BspServiceWorker<I extends WritableComparable, workerGraphPartitioner.finalizePartitionStats( partitionStatsList, getPartitionStore()); - return finishSuperstep(partitionStatsList); + return finishSuperstep(partitionStatsList, null); } /** @@ -730,7 +730,8 @@ public class BspServiceWorker<I extends WritableComparable, @Override public FinishedSuperstepStats finishSuperstep( - List<PartitionStats> partitionStatsList) { + List<PartitionStats> partitionStatsList, + GiraphTimerContext superstepTimerContext) { // This barrier blocks until success (or the master signals it to // restart). // @@ -769,6 +770,9 @@ public class BspServiceWorker<I extends WritableComparable, MemoryUtils.getRuntimeMemoryStats()); } + if (superstepTimerContext != null) { + superstepTimerContext.stop(); + } writeFinshedSuperstepInfoToZK(partitionStatsList, workerSentMessages, workerSentMessageBytes); http://git-wip-us.apache.org/repos/asf/giraph/blob/41944501/giraph-dist/pom.xml ---------------------------------------------------------------------- diff --git a/giraph-dist/pom.xml b/giraph-dist/pom.xml index 1830ca7..597a1d3 100644 --- a/giraph-dist/pom.xml +++ b/giraph-dist/pom.xml @@ -79,10 +79,6 @@ </dependency> <dependency> <groupId>org.apache.giraph</groupId> - <artifactId>giraph-rexster-io</artifactId> - </dependency> - <dependency> - <groupId>org.apache.giraph</groupId> <artifactId>giraph-hcatalog</artifactId> </dependency> <dependency>
