Updated Branches: refs/heads/trunk 768b7917f -> b3eada3c0
GIRAPH-773 Project: http://git-wip-us.apache.org/repos/asf/giraph/repo Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/b3eada3c Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/b3eada3c Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/b3eada3c Branch: refs/heads/trunk Commit: b3eada3c06deb0ca7137cad6ba8285be9d9df52c Parents: 768b791 Author: Claudio Martella <[email protected]> Authored: Tue Oct 15 23:33:11 2013 +0200 Committer: Claudio Martella <[email protected]> Committed: Tue Oct 15 23:33:11 2013 +0200 ---------------------------------------------------------------------- CHANGELOG | 6 ++++-- .../org/apache/giraph/bsp/BspOutputFormat.java | 19 +++++++++++++---- .../org/apache/giraph/conf/GiraphConstants.java | 2 +- .../job/GiraphConfigurationValidator.java | 22 ++++++++++++++++++-- 4 files changed, 40 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/giraph/blob/b3eada3c/CHANGELOG ---------------------------------------------------------------------- diff --git a/CHANGELOG b/CHANGELOG index 3df3164..f457026 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,8 @@ Giraph Change Log Release 1.1.0 - unreleased + GIRAPH-773: Cannot use only EdgeOutputFormat (armax00 via claudio) + GIRAPH-779: Make it easier to reuse objects with hive-io input (majakabiljo) GIRAPH-778: Testing with TestGraph is broken (majakabiljo) @@ -27,9 +29,9 @@ Release 1.1.0 - unreleased GIRAPH-746: Track and log versions of dependencies (nitay) GIRAPH-751: Build error: convertEdgeToLine in two different classes have the - same erasure, yet neither overrides the other (aarmax0 via claudio) + same erasure, yet neither overrides the other (armax00 via claudio) - GIRAPH-732: EdgeOutputFormat (aarmax00 via claudio) + GIRAPH-732: EdgeOutputFormat (armax00 via claudio) GIRAPH-512: JavaDoc warnings (tdn120 via nitay) http://git-wip-us.apache.org/repos/asf/giraph/blob/b3eada3c/giraph-core/src/main/java/org/apache/giraph/bsp/BspOutputFormat.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/bsp/BspOutputFormat.java b/giraph-core/src/main/java/org/apache/giraph/bsp/BspOutputFormat.java index 7e7c65f..cdb82b2 100644 --- a/giraph-core/src/main/java/org/apache/giraph/bsp/BspOutputFormat.java +++ b/giraph-core/src/main/java/org/apache/giraph/bsp/BspOutputFormat.java @@ -42,12 +42,18 @@ public class BspOutputFormat extends OutputFormat<Text, Text> { throws IOException, InterruptedException { ImmutableClassesGiraphConfiguration conf = new ImmutableClassesGiraphConfiguration(context.getConfiguration()); - if (!conf.hasVertexOutputFormat()) { + if (!conf.hasVertexOutputFormat() && !conf.hasEdgeOutputFormat()) { LOG.warn("checkOutputSpecs: ImmutableOutputCommiter" + " will not check anything"); return; } - conf.createWrappedVertexOutputFormat().checkOutputSpecs(context); + + if (conf.hasVertexOutputFormat()) { + conf.createWrappedVertexOutputFormat().checkOutputSpecs(context); + } + if (conf.hasEdgeOutputFormat()) { + conf.createWrappedEdgeOutputFormat().checkOutputSpecs(context); + } } @Override @@ -55,12 +61,17 @@ public class BspOutputFormat extends OutputFormat<Text, Text> { throws IOException, InterruptedException { ImmutableClassesGiraphConfiguration conf = new ImmutableClassesGiraphConfiguration(context.getConfiguration()); - if (!conf.hasVertexOutputFormat()) { + if (!conf.hasVertexOutputFormat() && !conf.hasEdgeOutputFormat()) { LOG.warn("getOutputCommitter: Returning " + "ImmutableOutputCommiter (does nothing)."); return new ImmutableOutputCommitter(); } - return conf.createWrappedVertexOutputFormat().getOutputCommitter(context); + + if (conf.hasVertexOutputFormat()) { + return conf.createWrappedVertexOutputFormat().getOutputCommitter(context); + } else { + return conf.createWrappedEdgeOutputFormat().getOutputCommitter(context); + } } @Override http://git-wip-us.apache.org/repos/asf/giraph/blob/b3eada3c/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java b/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java index 89fce61..eb8eb21 100644 --- a/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java +++ b/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java @@ -229,7 +229,7 @@ public interface GiraphConstants { EdgeOutputFormat.class, "EdgeOutputFormat class"); /** EdgeOutputFormat sub-directory */ StrConfOption EDGE_OUTPUT_FORMAT_SUBDIR = - new StrConfOption("giraph.edge.output.subdir", "edges", + new StrConfOption("giraph.edge.output.subdir", "", "EdgeOutputFormat sub-directory"); /** GiraphTextOuputFormat Separator */ http://git-wip-us.apache.org/repos/asf/giraph/blob/b3eada3c/giraph-core/src/main/java/org/apache/giraph/job/GiraphConfigurationValidator.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/job/GiraphConfigurationValidator.java b/giraph-core/src/main/java/org/apache/giraph/job/GiraphConfigurationValidator.java index fcb5b87..73a7aab 100644 --- a/giraph-core/src/main/java/org/apache/giraph/job/GiraphConfigurationValidator.java +++ b/giraph-core/src/main/java/org/apache/giraph/job/GiraphConfigurationValidator.java @@ -28,6 +28,7 @@ import org.apache.giraph.graph.DefaultVertexResolver; import org.apache.giraph.graph.VertexValueCombiner; import org.apache.giraph.graph.VertexResolver; import org.apache.giraph.io.EdgeInputFormat; +import org.apache.giraph.io.EdgeOutputFormat; import org.apache.giraph.io.VertexInputFormat; import org.apache.giraph.io.VertexOutputFormat; import org.apache.hadoop.conf.Configuration; @@ -140,6 +141,7 @@ public class GiraphConfigurationValidator<I extends WritableComparable, verifyVertexInputFormatGenericTypes(); verifyEdgeInputFormatGenericTypes(); verifyVertexOutputFormatGenericTypes(); + verifyEdgeOutputFormatGenericTypes(); verifyVertexResolverGenericTypes(); verifyVertexValueCombinerGenericTypes(); verifyMessageCombinerGenericTypes(); @@ -276,11 +278,27 @@ public class GiraphConfigurationValidator<I extends WritableComparable, } } - /** Verify that the output format's generic params match the job. */ + /** Verify that the vertex output format's generic params match the job. */ private void verifyVertexOutputFormatGenericTypes() { + Class<? extends EdgeOutputFormat<I, V, E>> + edgeOutputFormatClass = conf.getEdgeOutputFormatClass(); + if (conf.hasEdgeOutputFormat()) { + Class<?>[] classList = + getTypeArguments(EdgeOutputFormat.class, edgeOutputFormatClass); + checkAssignable(classList, ID_PARAM_INDEX, vertexIndexType(), + VertexOutputFormat.class, "vertex index"); + checkAssignable(classList, VALUE_PARAM_INDEX, vertexValueType(), + VertexOutputFormat.class, "vertex value"); + checkAssignable(classList, EDGE_PARAM_INDEX, edgeValueType(), + VertexOutputFormat.class, "edge value"); + } + } + + /** Verify that the edge output format's generic params match the job. */ + private void verifyEdgeOutputFormatGenericTypes() { Class<? extends VertexOutputFormat<I, V, E>> vertexOutputFormatClass = conf.getVertexOutputFormatClass(); - if (vertexOutputFormatClass != null) { + if (conf.hasVertexOutputFormat()) { Class<?>[] classList = getTypeArguments(VertexOutputFormat.class, vertexOutputFormatClass); checkAssignable(classList, ID_PARAM_INDEX, vertexIndexType(),
