Updated Branches: refs/heads/trunk 89445670e -> 8fbade6a7
http://git-wip-us.apache.org/repos/asf/giraph/blob/8fbade6a/giraph-hive/src/test/java/org/apache/giraph/hive/output/CheckOutputTest.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/output/CheckOutputTest.java b/giraph-hive/src/test/java/org/apache/giraph/hive/output/CheckOutputTest.java new file mode 100644 index 0000000..665600c --- /dev/null +++ b/giraph-hive/src/test/java/org/apache/giraph/hive/output/CheckOutputTest.java @@ -0,0 +1,90 @@ +package org.apache.giraph.hive.output; + +import org.apache.giraph.conf.GiraphConfiguration; +import org.apache.giraph.edge.ByteArrayEdges; +import org.apache.giraph.hive.GiraphHiveTestBase; +import org.apache.giraph.hive.common.GiraphHiveConstants; +import org.apache.giraph.hive.computations.ComputationCountEdges; +import org.apache.giraph.hive.output.examples.HiveOutputIntIntVertex; +import org.apache.giraph.io.formats.IntNullTextEdgeInputFormat; +import org.apache.giraph.utils.InternalVertexRunner; +import org.apache.thrift.TException; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.facebook.hiveio.common.HiveMetastores; +import com.facebook.hiveio.common.HiveType; +import com.facebook.hiveio.output.HiveOutputDescription; +import com.facebook.hiveio.schema.HiveTableSchema; +import com.facebook.hiveio.schema.TestSchema; +import com.facebook.hiveio.testing.LocalHiveServer; + +import java.io.IOException; + +import static com.facebook.hiveio.record.HiveRecordFactory.newWritableRecord; +import static org.junit.Assert.assertNull; + +public class CheckOutputTest extends GiraphHiveTestBase { + private LocalHiveServer hiveServer = new LocalHiveServer("giraph-hive"); + + @Before + public void setUp() throws IOException, TException { + hiveServer.init(); + HiveMetastores.setTestClient(hiveServer.getClient()); + } + + @Test + public void testCheck() throws Exception { + VertexToHive vertexToHive = new HiveOutputIntIntVertex(); + HiveOutputDescription outputDesc = new HiveOutputDescription(); + HiveTableSchema schema = TestSchema.builder() + .addColumn("foo", HiveType.LONG) + .addColumn("bar", HiveType.LONG) + .build(); + vertexToHive.checkOutput(outputDesc, schema, newWritableRecord(schema)); + + schema = TestSchema.builder() + .addColumn("foo", HiveType.INT) + .addColumn("bar", HiveType.LONG) + .build(); + checkThrows(vertexToHive, outputDesc, schema); + } + + private void checkThrows(VertexToHive vertexToHive, + HiveOutputDescription outputDesc, HiveTableSchema schema) { + try { + vertexToHive.checkOutput(outputDesc, schema, newWritableRecord(schema)); + } catch (IllegalArgumentException e) { + return; + } + Assert.fail(); + } + + @Test + public void testCheckFailsJob() throws Exception { + String tableName = "test1"; + hiveServer.createTable("CREATE TABLE " + tableName + + " (i1 INT, i2 BIGINT) "); + + GiraphConfiguration conf = new GiraphConfiguration(); + String[] edges = new String[] { + "1 2", + "2 3", + "2 4", + "4 1" + }; + + GiraphHiveConstants.HIVE_VERTEX_OUTPUT_TABLE.set(conf, tableName); + GiraphHiveConstants.VERTEX_TO_HIVE_CLASS.set(conf, HiveOutputIntIntVertex.class); + + conf.setComputationClass(ComputationCountEdges.class); + conf.setOutEdgesClass(ByteArrayEdges.class); + conf.setEdgeInputFormatClass(IntNullTextEdgeInputFormat.class); + conf.setVertexOutputFormatClass(HiveVertexOutputFormat.class); + try { + Iterable<String> result = InternalVertexRunner.run(conf, null, edges); + assertNull(result); + } catch (IllegalArgumentException e) { } + } +} http://git-wip-us.apache.org/repos/asf/giraph/blob/8fbade6a/giraph-hive/src/test/java/org/apache/giraph/hive/output/HiveOutputTest.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/output/HiveOutputTest.java b/giraph-hive/src/test/java/org/apache/giraph/hive/output/HiveOutputTest.java index 43e853b..1f92176 100644 --- a/giraph-hive/src/test/java/org/apache/giraph/hive/output/HiveOutputTest.java +++ b/giraph-hive/src/test/java/org/apache/giraph/hive/output/HiveOutputTest.java @@ -18,22 +18,14 @@ package org.apache.giraph.hive.output; import org.apache.giraph.conf.GiraphConfiguration; -import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration; import org.apache.giraph.edge.ByteArrayEdges; +import org.apache.giraph.hive.GiraphHiveTestBase; import org.apache.giraph.hive.Helpers; import org.apache.giraph.hive.common.GiraphHiveConstants; import org.apache.giraph.hive.computations.ComputationCountEdges; import org.apache.giraph.hive.output.examples.HiveOutputIntIntVertex; import org.apache.giraph.io.formats.IntNullTextEdgeInputFormat; -import org.apache.giraph.io.internal.WrappedVertexOutputFormat; import org.apache.giraph.utils.InternalVertexRunner; -import org.apache.hadoop.mapred.HackJobContext; -import org.apache.hadoop.mapred.HackTaskAttemptContext; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.TaskAttemptID; -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.mapreduce.OutputCommitter; -import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.thrift.TException; import org.junit.Before; import org.junit.BeforeClass; @@ -51,14 +43,9 @@ import java.util.Map; import static org.junit.Assert.assertEquals; -public class HiveOutputTest { +public class HiveOutputTest extends GiraphHiveTestBase { private LocalHiveServer hiveServer = new LocalHiveServer("giraph-hive"); - @BeforeClass - public static void hushDatanucleusWarnings() { - Helpers.silenceDataNucleusLogger(); - } - @Before public void setUp() throws IOException, TException { hiveServer.init(); @@ -76,7 +63,7 @@ public class HiveOutputTest { runJob(tableName, conf); HiveInputDescription inputDesc = new HiveInputDescription(); - inputDesc.setTableName(tableName); + inputDesc.getTableDesc().setTableName(tableName); verifyRecords(inputDesc); } @@ -95,7 +82,7 @@ public class HiveOutputTest { runJob(tableName, conf); HiveInputDescription inputDesc = new HiveInputDescription(); - inputDesc.setTableName(tableName); + inputDesc.getTableDesc().setTableName(tableName); inputDesc.setPartitionFilter("ds='foobar'"); verifyRecords(inputDesc); @@ -118,19 +105,7 @@ public class HiveOutputTest { conf.setVertexOutputFormatClass(HiveVertexOutputFormat.class); InternalVertexRunner.run(conf, null, edges); - commitJob(conf); - } - - private void commitJob(GiraphConfiguration conf) - throws IOException, InterruptedException { - ImmutableClassesGiraphConfiguration iconf = new ImmutableClassesGiraphConfiguration(conf); - WrappedVertexOutputFormat outputFormat = iconf.createWrappedVertexOutputFormat(); - JobConf jobConf = new JobConf(conf); - TaskAttemptContext taskContext = new HackTaskAttemptContext(jobConf, new TaskAttemptID()); - OutputCommitter outputCommitter = outputFormat.getOutputCommitter( - taskContext); - JobContext jobContext = new HackJobContext(jobConf, taskContext.getJobID()); - outputCommitter.commitJob(jobContext); + Helpers.commitJob(conf); } private void verifyRecords(HiveInputDescription inputDesc) http://git-wip-us.apache.org/repos/asf/giraph/blob/8fbade6a/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 7e166c4..82e9bdf 100644 --- a/pom.xml +++ b/pom.xml @@ -254,7 +254,7 @@ under the License. <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <dep.hiveio.version>0.11</dep.hiveio.version> + <dep.hiveio.version>0.14</dep.hiveio.version> <dep.jython.version>2.5.3</dep.jython.version> <dep.typetools.version>0.2.1</dep.typetools.version>
