Updated Branches: refs/heads/trunk 32b7f593f -> 75b311266
GIRAPH-636: Initialize compute OutEdges directly from input OutEdges (majakabiljo) Project: http://git-wip-us.apache.org/repos/asf/giraph/repo Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/75b31126 Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/75b31126 Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/75b31126 Branch: refs/heads/trunk Commit: 75b3112661c0900d7447d8ad56647b3e73ec0a24 Parents: 32b7f59 Author: Maja Kabiljo <[email protected]> Authored: Thu Apr 18 13:01:14 2013 -0700 Committer: Maja Kabiljo <[email protected]> Committed: Thu Apr 18 13:02:34 2013 -0700 ---------------------------------------------------------------------- CHANGELOG | 3 + .../java/org/apache/giraph/edge/EdgeStore.java | 7 +-- .../java/org/apache/giraph/edge/HashMapEdges.java | 14 +---- .../org/apache/giraph/edge/HashMultimapEdges.java | 14 +---- .../org/apache/giraph/edge/IntNullArrayEdges.java | 15 +---- .../apache/giraph/edge/LongDoubleArrayEdges.java | 16 +---- .../apache/giraph/edge/LongDoubleHashMapEdges.java | 15 +---- .../org/apache/giraph/edge/LongNullArrayEdges.java | 16 +---- .../apache/giraph/edge/LongNullHashSetEdges.java | 15 +---- .../org/apache/giraph/utils/EdgeIterables.java | 47 +++++++++++++++ 10 files changed, 71 insertions(+), 91 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/giraph/blob/75b31126/CHANGELOG ---------------------------------------------------------------------- diff --git a/CHANGELOG b/CHANGELOG index c1a2c85..d7679a2 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,9 @@ Giraph Change Log Release 1.0.1 - unreleased + GIRAPH-636: Initialize compute OutEdges directly from input OutEdges + (majakabiljo) + GIRAPH-645: Specify maven deploy plugin version (nitay) GIRAPH-643: Maven Release settings more friendly to Git (nitay) http://git-wip-us.apache.org/repos/asf/giraph/blob/75b31126/giraph-core/src/main/java/org/apache/giraph/edge/EdgeStore.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/edge/EdgeStore.java b/giraph-core/src/main/java/org/apache/giraph/edge/EdgeStore.java index 81044bd..9e2d246 100644 --- a/giraph-core/src/main/java/org/apache/giraph/edge/EdgeStore.java +++ b/giraph-core/src/main/java/org/apache/giraph/edge/EdgeStore.java @@ -146,12 +146,7 @@ public class EdgeStore<I extends WritableComparable, if (!useInputOutEdges) { return inputEdges; } else { - OutEdges<I, E> computeEdges = - configuration.createAndInitializeOutEdges(inputEdges.size()); - for (Edge<I, E> edge : inputEdges) { - computeEdges.add(edge); - } - return computeEdges; + return configuration.createAndInitializeOutEdges(inputEdges); } } http://git-wip-us.apache.org/repos/asf/giraph/blob/75b31126/giraph-core/src/main/java/org/apache/giraph/edge/HashMapEdges.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/edge/HashMapEdges.java b/giraph-core/src/main/java/org/apache/giraph/edge/HashMapEdges.java index fc23cf0..0945634 100644 --- a/giraph-core/src/main/java/org/apache/giraph/edge/HashMapEdges.java +++ b/giraph-core/src/main/java/org/apache/giraph/edge/HashMapEdges.java @@ -19,13 +19,14 @@ package org.apache.giraph.edge; import com.google.common.collect.Maps; + +import org.apache.giraph.utils.EdgeIterables; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.Map; @@ -48,16 +49,7 @@ public class HashMapEdges<I extends WritableComparable, E extends Writable> @Override public void initialize(Iterable<Edge<I, E>> edges) { - // If the iterable is actually a collection, we can cheaply get the - // size and initialize the hash-map with the expected capacity. - if (edges instanceof Collection) { - initialize(((Collection<Edge<I, E>>) edges).size()); - } else { - initialize(); - } - for (Edge<I, E> edge : edges) { - add(edge); - } + EdgeIterables.initialize(this, edges); } @Override http://git-wip-us.apache.org/repos/asf/giraph/blob/75b31126/giraph-core/src/main/java/org/apache/giraph/edge/HashMultimapEdges.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/edge/HashMultimapEdges.java b/giraph-core/src/main/java/org/apache/giraph/edge/HashMultimapEdges.java index abe6151..e5e39de 100644 --- a/giraph-core/src/main/java/org/apache/giraph/edge/HashMultimapEdges.java +++ b/giraph-core/src/main/java/org/apache/giraph/edge/HashMultimapEdges.java @@ -20,13 +20,14 @@ package org.apache.giraph.edge; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.UnmodifiableIterator; + +import org.apache.giraph.utils.EdgeIterables; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.util.Collection; import java.util.Iterator; import java.util.Map; @@ -47,16 +48,7 @@ public class HashMultimapEdges<I extends WritableComparable, E extends Writable> @Override public void initialize(Iterable<Edge<I, E>> edges) { - // If the iterable is actually a collection, we can cheaply get the - // size and initialize the hash-multimap with the expected capacity. - if (edges instanceof Collection) { - initialize(((Collection<Edge<I, E>>) edges).size()); - } else { - initialize(); - } - for (Edge<I, E> edge : edges) { - add(edge); - } + EdgeIterables.initialize(this, edges); } /** http://git-wip-us.apache.org/repos/asf/giraph/blob/75b31126/giraph-core/src/main/java/org/apache/giraph/edge/IntNullArrayEdges.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/edge/IntNullArrayEdges.java b/giraph-core/src/main/java/org/apache/giraph/edge/IntNullArrayEdges.java index 9867771..e54b41f 100644 --- a/giraph-core/src/main/java/org/apache/giraph/edge/IntNullArrayEdges.java +++ b/giraph-core/src/main/java/org/apache/giraph/edge/IntNullArrayEdges.java @@ -18,6 +18,7 @@ package org.apache.giraph.edge; +import org.apache.giraph.utils.EdgeIterables; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.NullWritable; @@ -29,7 +30,6 @@ import it.unimi.dsi.fastutil.ints.IntIterator; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.util.Collection; import java.util.Iterator; /** @@ -46,18 +46,7 @@ public class IntNullArrayEdges @Override public void initialize(Iterable<Edge<IntWritable, NullWritable>> edges) { - // If the iterable is actually a collection, we can cheaply get the - // size and initialize the array with the expected capacity. - if (edges instanceof Collection) { - int numEdges = - ((Collection<Edge<IntWritable, NullWritable>>) edges).size(); - initialize(numEdges); - } else { - initialize(); - } - for (Edge<IntWritable, NullWritable> edge : edges) { - add(edge); - } + EdgeIterables.initialize(this, edges); } @Override http://git-wip-us.apache.org/repos/asf/giraph/blob/75b31126/giraph-core/src/main/java/org/apache/giraph/edge/LongDoubleArrayEdges.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/edge/LongDoubleArrayEdges.java b/giraph-core/src/main/java/org/apache/giraph/edge/LongDoubleArrayEdges.java index 45d182e..324a32c 100644 --- a/giraph-core/src/main/java/org/apache/giraph/edge/LongDoubleArrayEdges.java +++ b/giraph-core/src/main/java/org/apache/giraph/edge/LongDoubleArrayEdges.java @@ -23,13 +23,14 @@ import it.unimi.dsi.fastutil.doubles.DoubleArrayList; import it.unimi.dsi.fastutil.doubles.DoubleIterator; import it.unimi.dsi.fastutil.longs.LongArrayList; import it.unimi.dsi.fastutil.longs.LongIterator; + +import org.apache.giraph.utils.EdgeIterables; import org.apache.hadoop.io.DoubleWritable; import org.apache.hadoop.io.LongWritable; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.util.Collection; import java.util.Iterator; /** @@ -49,18 +50,7 @@ public class LongDoubleArrayEdges @Override public void initialize(Iterable<Edge<LongWritable, DoubleWritable>> edges) { - // If the iterable is actually a collection, we can cheaply get the - // size and initialize the arrays with the expected capacity. - if (edges instanceof Collection) { - int numEdges = - ((Collection<Edge<LongWritable, DoubleWritable>>) edges).size(); - initialize(numEdges); - } else { - initialize(); - } - for (Edge<LongWritable, DoubleWritable> edge : edges) { - add(edge); - } + EdgeIterables.initialize(this, edges); } @Override http://git-wip-us.apache.org/repos/asf/giraph/blob/75b31126/giraph-core/src/main/java/org/apache/giraph/edge/LongDoubleHashMapEdges.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/edge/LongDoubleHashMapEdges.java b/giraph-core/src/main/java/org/apache/giraph/edge/LongDoubleHashMapEdges.java index c33200d..e6c8229 100644 --- a/giraph-core/src/main/java/org/apache/giraph/edge/LongDoubleHashMapEdges.java +++ b/giraph-core/src/main/java/org/apache/giraph/edge/LongDoubleHashMapEdges.java @@ -22,13 +22,14 @@ import com.google.common.collect.UnmodifiableIterator; import it.unimi.dsi.fastutil.longs.Long2DoubleMap; import it.unimi.dsi.fastutil.longs.Long2DoubleOpenHashMap; import it.unimi.dsi.fastutil.objects.ObjectIterator; + +import org.apache.giraph.utils.EdgeIterables; import org.apache.hadoop.io.DoubleWritable; import org.apache.hadoop.io.LongWritable; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.util.Collection; import java.util.Iterator; /** @@ -50,17 +51,7 @@ public class LongDoubleHashMapEdges @Override public void initialize(Iterable<Edge<LongWritable, DoubleWritable>> edges) { - // If the iterable is actually a collection, we can cheaply get the - // size and initialize the hash-map with the expected capacity. - if (edges instanceof Collection) { - initialize( - ((Collection<Edge<LongWritable, DoubleWritable>>) edges).size()); - } else { - initialize(); - } - for (Edge<LongWritable, DoubleWritable> edge : edges) { - add(edge); - } + EdgeIterables.initialize(this, edges); } @Override http://git-wip-us.apache.org/repos/asf/giraph/blob/75b31126/giraph-core/src/main/java/org/apache/giraph/edge/LongNullArrayEdges.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/edge/LongNullArrayEdges.java b/giraph-core/src/main/java/org/apache/giraph/edge/LongNullArrayEdges.java index d753588..b7b3af5 100644 --- a/giraph-core/src/main/java/org/apache/giraph/edge/LongNullArrayEdges.java +++ b/giraph-core/src/main/java/org/apache/giraph/edge/LongNullArrayEdges.java @@ -20,13 +20,14 @@ package org.apache.giraph.edge; import it.unimi.dsi.fastutil.longs.LongArrayList; import it.unimi.dsi.fastutil.longs.LongIterator; + +import org.apache.giraph.utils.EdgeIterables; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.NullWritable; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.util.Collection; import java.util.Iterator; /** @@ -44,18 +45,7 @@ public class LongNullArrayEdges @Override public void initialize(Iterable<Edge<LongWritable, NullWritable>> edges) { - // If the iterable is actually a collection, we can cheaply get the - // size and initialize the arrays with the expected capacity. - if (edges instanceof Collection) { - int numEdges = - ((Collection<Edge<LongWritable, NullWritable>>) edges).size(); - initialize(numEdges); - } else { - initialize(); - } - for (Edge<LongWritable, NullWritable> edge : edges) { - add(edge); - } + EdgeIterables.initialize(this, edges); } @Override http://git-wip-us.apache.org/repos/asf/giraph/blob/75b31126/giraph-core/src/main/java/org/apache/giraph/edge/LongNullHashSetEdges.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/edge/LongNullHashSetEdges.java b/giraph-core/src/main/java/org/apache/giraph/edge/LongNullHashSetEdges.java index ef0302c..139fc7a 100644 --- a/giraph-core/src/main/java/org/apache/giraph/edge/LongNullHashSetEdges.java +++ b/giraph-core/src/main/java/org/apache/giraph/edge/LongNullHashSetEdges.java @@ -20,13 +20,14 @@ package org.apache.giraph.edge; import it.unimi.dsi.fastutil.longs.LongIterator; import it.unimi.dsi.fastutil.longs.LongOpenHashSet; + +import org.apache.giraph.utils.EdgeIterables; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.NullWritable; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.util.Collection; import java.util.Iterator; /** @@ -45,17 +46,7 @@ public class LongNullHashSetEdges @Override public void initialize(Iterable<Edge<LongWritable, NullWritable>> edges) { - // If the iterable is actually a collection, we can cheaply get the - // size and initialize the hash-map with the expected capacity. - if (edges instanceof Collection) { - initialize( - ((Collection<Edge<LongWritable, NullWritable>>) edges).size()); - } else { - initialize(); - } - for (Edge<LongWritable, NullWritable> edge : edges) { - add(edge); - } + EdgeIterables.initialize(this, edges); } @Override http://git-wip-us.apache.org/repos/asf/giraph/blob/75b31126/giraph-core/src/main/java/org/apache/giraph/utils/EdgeIterables.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/utils/EdgeIterables.java b/giraph-core/src/main/java/org/apache/giraph/utils/EdgeIterables.java index e945d7b..0d2b810 100644 --- a/giraph-core/src/main/java/org/apache/giraph/utils/EdgeIterables.java +++ b/giraph-core/src/main/java/org/apache/giraph/utils/EdgeIterables.java @@ -21,11 +21,14 @@ package org.apache.giraph.utils; import com.google.common.collect.Iterables; import org.apache.giraph.edge.Edge; import org.apache.giraph.edge.EdgeFactory; +import org.apache.giraph.edge.OutEdges; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.io.WritableUtils; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.Iterator; @@ -105,4 +108,48 @@ public class EdgeIterables { Collections.sort(edgeList2, edgeComparator); return equals(edgeList1, edgeList2); } + + /** + * Get the size of edges. Optimized implementation for cases when edges is + * instance of {@link OutEdges} or {@link Collection} which only calls + * size() method from these classes. + * + * @param edges Edges + * @param <I> Vertex index + * @param <E> Edge value + * @return Size of edges + */ + public static <I extends WritableComparable, E extends Writable> int size( + Iterable<Edge<I, E>> edges) { + if (edges instanceof OutEdges) { + return ((OutEdges) edges).size(); + } else { + return Iterables.size(edges); + } + } + + /** + * Initialize edges data structure and add the edges from edgesIterable. + * + * If edgesIterable is instance of {@link OutEdges} or {@link Collection} + * edges will be initialized with size of edgesIterable, + * otherwise edges will be initialized without size. + * + * @param edges Edges to initialize + * @param edgesIterable Iterable whose edges to use + * @param <I> Vertex index + * @param <E> Edge value + */ + public static <I extends WritableComparable, E extends Writable> + void initialize(OutEdges<I, E> edges, Iterable<Edge<I, E>> edgesIterable) { + if (edgesIterable instanceof OutEdges || + edgesIterable instanceof Collection) { + edges.initialize(size(edgesIterable)); + } else { + edges.initialize(); + } + for (Edge<I, E> edge : edgesIterable) { + edges.add(edge); + } + } }
