fix schema issue
Project: http://git-wip-us.apache.org/repos/asf/giraph/repo Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/8fe2a03c Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/8fe2a03c Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/8fe2a03c Branch: refs/heads/perf Commit: 8fe2a03cf964158270ffba7e69777230968b3445 Parents: 9a7c55f Author: Nitay Joffe <[email protected]> Authored: Mon Mar 25 00:36:26 2013 -0400 Committer: Nitay Joffe <[email protected]> Committed: Mon Mar 25 00:36:26 2013 -0400 ---------------------------------------------------------------------- .../org/apache/giraph/hive/HiveGiraphRunner.java | 22 +++++++++++---- .../apache/giraph/hive/common/HiveProfiles.java | 3 -- pom.xml | 2 +- 3 files changed, 17 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/giraph/blob/8fe2a03c/giraph-hive/src/main/java/org/apache/giraph/hive/HiveGiraphRunner.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/HiveGiraphRunner.java b/giraph-hive/src/main/java/org/apache/giraph/hive/HiveGiraphRunner.java index 4b6cab9..5233709 100644 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/HiveGiraphRunner.java +++ b/giraph-hive/src/main/java/org/apache/giraph/hive/HiveGiraphRunner.java @@ -26,19 +26,18 @@ import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.giraph.conf.GiraphClasses; import org.apache.giraph.conf.GiraphConfiguration; -import org.apache.giraph.hive.common.HiveProfiles; +import org.apache.giraph.graph.Vertex; import org.apache.giraph.hive.input.edge.HiveEdgeInputFormat; import org.apache.giraph.hive.input.edge.HiveEdgeReader; import org.apache.giraph.hive.input.edge.HiveToEdge; -import org.apache.giraph.hive.input.vertex.HiveToVertexValue; import org.apache.giraph.hive.input.vertex.HiveToVertexEdges; +import org.apache.giraph.hive.input.vertex.HiveToVertexValue; import org.apache.giraph.hive.input.vertex.HiveVertexInputFormat; import org.apache.giraph.hive.input.vertex.HiveVertexReader; import org.apache.giraph.hive.output.HiveVertexOutputFormat; import org.apache.giraph.hive.output.HiveVertexWriter; import org.apache.giraph.hive.output.VertexToHive; import org.apache.giraph.job.GiraphJob; -import org.apache.giraph.graph.Vertex; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.util.Tool; @@ -50,6 +49,7 @@ import com.facebook.giraph.hive.input.HiveApiInputFormat; import com.facebook.giraph.hive.input.HiveInputDescription; import com.facebook.giraph.hive.output.HiveApiOutputFormat; import com.facebook.giraph.hive.output.HiveOutputDescription; +import com.facebook.giraph.hive.schema.HiveTableSchemas; import com.google.common.base.Splitter; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -60,6 +60,10 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import static org.apache.giraph.hive.common.HiveProfiles.EDGE_INPUT_PROFILE_ID; +import static org.apache.giraph.hive.common.HiveProfiles.VERTEX_INPUT_PROFILE_ID; +import static org.apache.giraph.hive.common.HiveProfiles.VERTEX_OUTPUT_PROFILE_ID; + /** * Hive Giraph Runner */ @@ -276,14 +280,18 @@ public class HiveGiraphRunner implements Tool { private void setupHiveInputs(GiraphConfiguration conf) throws TException { if (hiveToVertexClass != null) { HiveApiInputFormat.setProfileInputDesc(conf, hiveVertexInputDescription, - HiveProfiles.VERTEX_INPUT_PROFILE_ID); + VERTEX_INPUT_PROFILE_ID); conf.setVertexInputFormatClass(HiveVertexInputFormat.class); + HiveTableSchemas.put(conf, VERTEX_INPUT_PROFILE_ID, + hiveVertexInputDescription.hiveTableName()); } if (hiveToEdgeClass != null) { HiveApiInputFormat.setProfileInputDesc(conf, hiveEdgeInputDescription, - HiveProfiles.EDGE_INPUT_PROFILE_ID); + EDGE_INPUT_PROFILE_ID); conf.setEdgeInputFormatClass(HiveEdgeInputFormat.class); + HiveTableSchemas.put(conf, EDGE_INPUT_PROFILE_ID, + hiveEdgeInputDescription.hiveTableName()); } } @@ -298,8 +306,10 @@ public class HiveGiraphRunner implements Tool { LOG.warn("run: Warning - Output will be skipped!"); } else if (vertexToHiveClass != null) { HiveApiOutputFormat.initProfile(conf, hiveOutputDescription, - HiveProfiles.VERTEX_OUTPUT_PROFILE_ID); + VERTEX_OUTPUT_PROFILE_ID); conf.setVertexOutputFormatClass(HiveVertexOutputFormat.class); + HiveTableSchemas.put(conf, VERTEX_OUTPUT_PROFILE_ID, + hiveOutputDescription.hiveTableName()); } else { LOG.fatal("output requested but " + VertexToHive.class.getSimpleName() + " not set"); http://git-wip-us.apache.org/repos/asf/giraph/blob/8fe2a03c/giraph-hive/src/main/java/org/apache/giraph/hive/common/HiveProfiles.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/common/HiveProfiles.java b/giraph-hive/src/main/java/org/apache/giraph/hive/common/HiveProfiles.java index b0ddc48..892d443 100644 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/common/HiveProfiles.java +++ b/giraph-hive/src/main/java/org/apache/giraph/hive/common/HiveProfiles.java @@ -26,9 +26,6 @@ public class HiveProfiles { public static final String VERTEX_INPUT_PROFILE_ID = "vertex_input_profile"; /** name of edge input profile */ public static final String EDGE_INPUT_PROFILE_ID = "edge_input_profile"; - /** name of vertex value input profile */ - public static final String VERTEX_VALUE_INPUT_PROFILE_ID = - "vertex_value_input_profile"; /** Name of vertex output profile */ public static final String VERTEX_OUTPUT_PROFILE_ID = "vertex_output_profile"; http://git-wip-us.apache.org/repos/asf/giraph/blob/8fe2a03c/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index e576e4b..d94ce7a 100644 --- a/pom.xml +++ b/pom.xml @@ -881,7 +881,7 @@ under the License. <dependency> <groupId>com.facebook.giraph.hive</groupId> <artifactId>hive-io-experimental</artifactId> - <version>0.4-SNAPSHOT</version> + <version>0.5-SNAPSHOT</version> </dependency> <dependency> <groupId>com.google.guava</groupId>
