Updated Branches: refs/heads/trunk 7ea62d9c1 -> f8a3c777e
GIRAPH-696: Should be able to spill giraph metrics to a specified directory on HDFS Project: http://git-wip-us.apache.org/repos/asf/giraph/repo Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/f8a3c777 Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/f8a3c777 Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/f8a3c777 Branch: refs/heads/trunk Commit: f8a3c777e1fc1902a6a965c1d2a33e7ce3cc1643 Parents: 7ea62d9 Author: Claudio Martella <[email protected]> Authored: Tue Jun 25 23:59:32 2013 +0200 Committer: Claudio Martella <[email protected]> Committed: Tue Jun 25 23:59:32 2013 +0200 ---------------------------------------------------------------------- CHANGELOG | 3 ++ .../org/apache/giraph/conf/GiraphConstants.java | 5 +++ .../apache/giraph/master/BspServiceMaster.java | 41 +++++++++++++++++++- 3 files changed, 48 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/giraph/blob/f8a3c777/CHANGELOG ---------------------------------------------------------------------- diff --git a/CHANGELOG b/CHANGELOG index 36693ab..0f2758e 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,9 @@ Giraph Change Log Release 1.1.0 - unreleased + GIRAPH-696: Should be able to spill giraph metrics to a specified + directory on HDFS (claudio) + GIRAPH-695: Add getCombiner and getComputation methods for MasterCompute (aching) http://git-wip-us.apache.org/repos/asf/giraph/blob/f8a3c777/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java b/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java index 80afab9..c65b5f0 100644 --- a/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java +++ b/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java @@ -315,6 +315,11 @@ public interface GiraphConstants { new BooleanConfOption("giraph.metrics.enable", false, "Enable the Metrics system"); + /** Directory in HDFS to write master metrics to, instead of stderr */ + StrConfOption METRICS_DIRECTORY = + new StrConfOption("giraph.metrics.directory", "", + "Directory in HDFS to write master metrics to, instead of stderr"); + /** * ZooKeeper comma-separated list (if not set, * will start up ZooKeeper locally) http://git-wip-us.apache.org/repos/asf/giraph/blob/f8a3c777/giraph-core/src/main/java/org/apache/giraph/master/BspServiceMaster.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/master/BspServiceMaster.java b/giraph-core/src/main/java/org/apache/giraph/master/BspServiceMaster.java index 0d266a6..3558887 100644 --- a/giraph-core/src/main/java/org/apache/giraph/master/BspServiceMaster.java +++ b/giraph-core/src/main/java/org/apache/giraph/master/BspServiceMaster.java @@ -91,6 +91,7 @@ import java.io.DataInputStream; import java.io.DataOutput; import java.io.DataOutputStream; import java.io.IOException; +import java.io.PrintStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -974,7 +975,11 @@ public class BspServiceMaster<I extends WritableComparable, } if (conf.metricsEnabled()) { - aggregatedMetrics.print(superstep, System.err); + if (GiraphConstants.METRICS_DIRECTORY.isDefaultValue(conf)) { + aggregatedMetrics.print(superstep, System.err); + } else { + printAggregatedMetricsToHDFS(superstep, aggregatedMetrics); + } } if (LOG.isInfoEnabled()) { @@ -985,6 +990,40 @@ public class BspServiceMaster<I extends WritableComparable, } /** + * Write superstep metrics to own file in HDFS + * @param superstep the current superstep + * @param aggregatedMetrics the aggregated metrics to write + */ + private void printAggregatedMetricsToHDFS( + long superstep, AggregatedMetrics aggregatedMetrics) { + ImmutableClassesGiraphConfiguration conf = getConfiguration(); + PrintStream out = null; + Path dir = new Path(GiraphConstants.METRICS_DIRECTORY.get(conf)); + Path outFile = new Path(GiraphConstants.METRICS_DIRECTORY.get(conf) + + Path.SEPARATOR_CHAR + "superstep_" + superstep + ".metrics"); + try { + FileSystem fs; + fs = FileSystem.get(conf); + if (!fs.exists(dir)) { + fs.mkdirs(dir); + } + if (fs.exists(outFile)) { + throw new RuntimeException( + "printAggregatedMetricsToHDFS: metrics file exists"); + } + out = new PrintStream(fs.create(outFile)); + aggregatedMetrics.print(superstep, out); + } catch (IOException e) { + throw new RuntimeException( + "printAggregatedMetricsToHDFS: error creating metrics file", e); + } finally { + if (out != null) { + out.close(); + } + } + } + + /** * Finalize the checkpoint file prefixes by taking the chosen workers and * writing them to a finalized file. Also write out the master * aggregated aggregator array from the previous superstep.
