Updated Branches: refs/heads/trunk 95e122676 -> f4deb5ca8
GIRAPH-597: Don't reuse vertex by default in SimpleHiveToVertex (majakabiljo) Project: http://git-wip-us.apache.org/repos/asf/giraph/repo Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/f4deb5ca Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/f4deb5ca Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/f4deb5ca Branch: refs/heads/trunk Commit: f4deb5ca809e8b1663301589e3c0cc94e370cd3d Parents: 95e1226 Author: Maja Kabiljo <[email protected]> Authored: Thu Mar 28 10:28:51 2013 -0700 Committer: Maja Kabiljo <[email protected]> Committed: Thu Mar 28 10:28:51 2013 -0700 ---------------------------------------------------------------------- CHANGELOG | 2 + .../giraph/hive/input/edge/SimpleHiveToEdge.java | 7 ++++ .../hive/input/vertex/SimpleHiveToVertex.java | 24 ++++++++++++-- 3 files changed, 29 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/giraph/blob/f4deb5ca/CHANGELOG ---------------------------------------------------------------------- diff --git a/CHANGELOG b/CHANGELOG index ff20214..8da6a12 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,8 @@ Giraph Change Log Release 0.2.0 - unreleased + GIRAPH-597: Don't reuse vertex by default in SimpleHiveToVertex (majakabiljo) + GIRAPH-588: More flexible Hive input (majakabiljo) GIRAPH-587: Refactor configuration options (nitay) http://git-wip-us.apache.org/repos/asf/giraph/blob/f4deb5ca/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/SimpleHiveToEdge.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/SimpleHiveToEdge.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/SimpleHiveToEdge.java index 0b76683..7aa8721 100644 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/SimpleHiveToEdge.java +++ b/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/SimpleHiveToEdge.java @@ -18,6 +18,7 @@ package org.apache.giraph.hive.input.edge; +import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration; import org.apache.giraph.io.iterables.EdgeWithSource; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; @@ -69,6 +70,12 @@ public abstract class SimpleHiveToEdge<I extends WritableComparable, public abstract E getEdgeValue(HiveReadableRecord hiveRecord); @Override + public void setConf(ImmutableClassesGiraphConfiguration<I, V, E, M> conf) { + super.setConf(conf); + reusableEdge.setEdge(getConf().createReusableEdge()); + } + + @Override public final void initializeRecords(Iterator<HiveRecord> records) { this.records = records; } http://git-wip-us.apache.org/repos/asf/giraph/blob/f4deb5ca/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/SimpleHiveToVertex.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/SimpleHiveToVertex.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/SimpleHiveToVertex.java index f42536d..651aefd 100644 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/SimpleHiveToVertex.java +++ b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/SimpleHiveToVertex.java @@ -18,6 +18,8 @@ package org.apache.giraph.hive.input.vertex; +import org.apache.giraph.conf.BooleanConfOption; +import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration; import org.apache.giraph.edge.Edge; import org.apache.giraph.graph.Vertex; import org.apache.hadoop.io.Writable; @@ -40,11 +42,14 @@ import java.util.Iterator; public abstract class SimpleHiveToVertex<I extends WritableComparable, V extends Writable, E extends Writable, M extends Writable> extends AbstractHiveToVertex<I, V, E, M> { + /** Configuration option for whether to reuse vertex */ + public static final BooleanConfOption REUSE_VERTEX_KEY = + new BooleanConfOption("giraph.hive.reuse.vertex", false); /** Hive records which we are reading from */ private Iterator<HiveRecord> records; /** Reusable vertex object */ - private Vertex<I, V, E, M> reusableVertex; + private Vertex<I, V, E, M> reusableVertex = null; /** * Read the Vertex's ID from the HiveRecord given. @@ -71,9 +76,16 @@ public abstract class SimpleHiveToVertex<I extends WritableComparable, public abstract Iterable<Edge<I, E>> getEdges(HiveReadableRecord record); @Override + public void setConf(ImmutableClassesGiraphConfiguration<I, V, E, M> conf) { + super.setConf(conf); + if (REUSE_VERTEX_KEY.get(conf)) { + reusableVertex = getConf().createVertex(); + } + } + + @Override public void initializeRecords(Iterator<HiveRecord> records) { this.records = records; - reusableVertex = getConf().createVertex(); } @Override @@ -87,7 +99,11 @@ public abstract class SimpleHiveToVertex<I extends WritableComparable, I id = getVertexId(record); V value = getVertexValue(record); Iterable<Edge<I, E>> edges = getEdges(record); - reusableVertex.initialize(id, value, edges); - return reusableVertex; + Vertex<I, V, E, M> vertex = reusableVertex; + if (vertex == null) { + vertex = getConf().createVertex(); + } + vertex.initialize(id, value, edges); + return vertex; } }
