Updated Branches: refs/heads/trunk a419a7bc4 -> fd9690a05
GIRAPH-655: Fix hadoop 2.0.0 build (nitay) Project: http://git-wip-us.apache.org/repos/asf/giraph/repo Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/fd9690a0 Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/fd9690a0 Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/fd9690a0 Branch: refs/heads/trunk Commit: fd9690a05de75707de59f5c44d56373bed500819 Parents: a419a7b Author: Nitay Joffe <[email protected]> Authored: Mon May 6 18:30:31 2013 -0400 Committer: Nitay Joffe <[email protected]> Committed: Tue May 7 00:03:37 2013 -0400 ---------------------------------------------------------------------- .gitignore | 1 + CHANGELOG | 2 + dev-support/for-each-profile.sh | 49 +++----- .../java/org/apache/giraph/job/HadoopUtils.java | 87 +++++++++++++++ .../test/java/org/apache/giraph/TestBspBasic.java | 30 ++---- .../apache/giraph/hive/output/HiveOutputTest.java | 11 +- pom.xml | 7 +- 7 files changed, 127 insertions(+), 60 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/giraph/blob/fd9690a0/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 8c911c1..42ecd00 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,5 @@ failed-profile.txt # Vim auto-save files: .*.swp +/for-each-profile-results.txt /giraph-hive/derby.log http://git-wip-us.apache.org/repos/asf/giraph/blob/fd9690a0/CHANGELOG ---------------------------------------------------------------------- diff --git a/CHANGELOG b/CHANGELOG index 954401c..e81a478 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,8 @@ Giraph Change Log Release 1.1.0 - unreleased + GIRAPH-655: Fix Hive Build for Hadoop 2.0.0 (nitay) + GIRAPH-660: Move the snapshot to 1.1.0 and also add a releases page for the 1.0 release (aching) http://git-wip-us.apache.org/repos/asf/giraph/blob/fd9690a0/dev-support/for-each-profile.sh ---------------------------------------------------------------------- diff --git a/dev-support/for-each-profile.sh b/dev-support/for-each-profile.sh index cefafb2..4917fb9 100755 --- a/dev-support/for-each-profile.sh +++ b/dev-support/for-each-profile.sh @@ -49,41 +49,28 @@ parse_profiles echo "Running on profiles: $all_profiles" -failed_profile_file=$TOP_DIR/failed-profile.txt -if [[ -r "$failed_profile_file" ]]; then - echo "Reading $failed_profile_file to restart from last run state." - echo "Remove that file if you want a fresh run" - failed_profile=$(cat $failed_profile_file) - run_profile=false -else - run_profile=true -fi +results_file=$TOP_DIR/for-each-profile-results.txt + +echo "mvn -P<profile> $@" > $results_file +echo "======================================================" >> $results_file for profile in $all_profiles; do - if ! $run_profile; then - if [[ "${profile}" == "${failed_profile}" ]]; then - run_profile=true - fi - fi - if $run_profile; then - echo_separator - echo "=== $profile ===" - echo_separator + echo_separator + echo "=== $profile ===" + echo_separator - mvn -P$profile $@ - result=$? - - if [[ $result -ne 0 ]]; then - echo_separator - echo "=== Failed on profile: $profile" - echo "=== Failed command: mvn -P$profile $@" - echo_separator - echo $profile > $failed_profile_file - exit $result - fi - else + args="-P$profile $@" + + mvn $args + result=$? + + if [[ $result -ne 0 ]]; then echo_separator - echo "=== Skipping profile $profile" + echo "=== Failed on profile: $profile" + echo "=== Failed command: mvn $args" + echo "$profile: FAILED" >> $results_file echo_separator + else + echo "$profile: SUCCESS" >> $results_file fi done http://git-wip-us.apache.org/repos/asf/giraph/blob/fd9690a0/giraph-core/src/main/java/org/apache/giraph/job/HadoopUtils.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/job/HadoopUtils.java b/giraph-core/src/main/java/org/apache/giraph/job/HadoopUtils.java new file mode 100644 index 0000000..d5095bc --- /dev/null +++ b/giraph-core/src/main/java/org/apache/giraph/job/HadoopUtils.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.giraph.job; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.JobID; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.TaskAttemptID; +import org.apache.hadoop.mapreduce.TaskAttemptContext; + +/*if_not[HADOOP_NON_JOBCONTEXT_IS_INTERFACE]*/ +import org.apache.hadoop.mapreduce.task.JobContextImpl; +import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; +/*end[HADOOP_NON_JOBCONTEXT_IS_INTERFACE]*/ + +/** + * Helpers for dealing with Hadoop. + */ +public class HadoopUtils { + /** Don't construct */ + private HadoopUtils() { } + + /** + * Create a TaskAttemptContext, supporting many Hadoops. + * + * @param conf Configuration + * @return TaskAttemptContext + */ + public static TaskAttemptContext makeTaskAttemptContext(Configuration conf) { + TaskAttemptContext context; + /*if[HADOOP_NON_JOBCONTEXT_IS_INTERFACE] + context = new TaskAttemptContext(conf, new TaskAttemptID()); + else[HADOOP_NON_JOBCONTEXT_IS_INTERFACE]*/ + context = new TaskAttemptContextImpl(conf, new TaskAttemptID()); + /*end[HADOOP_NON_JOBCONTEXT_IS_INTERFACE]*/ + return context; + } + + /** + * Create a TaskAttemptContext, supporting many Hadoops. + * + * @return TaskAttemptContext + */ + public static TaskAttemptContext makeTaskContext() { + return makeTaskAttemptContext(new Configuration()); + } + + /** + * Create a JobContext, supporting many Hadoops. + * + * @param conf Configuration + * @return JobContext + */ + public static JobContext makeJobContext(Configuration conf) { + JobContext context; + /*if[HADOOP_NON_JOBCONTEXT_IS_INTERFACE] + context = new JobContext(conf, new JobID()); + else[HADOOP_NON_JOBCONTEXT_IS_INTERFACE]*/ + context = new JobContextImpl(conf, new JobID()); + /*end[HADOOP_NON_JOBCONTEXT_IS_INTERFACE]*/ + return context; + } + + /** + * Create a JobContext, supporting many Hadoops. + * + * @return JobContext + */ + public static JobContext makeJobContext() { + return makeJobContext(new Configuration()); + } +} http://git-wip-us.apache.org/repos/asf/giraph/blob/fd9690a0/giraph-examples/src/test/java/org/apache/giraph/TestBspBasic.java ---------------------------------------------------------------------- diff --git a/giraph-examples/src/test/java/org/apache/giraph/TestBspBasic.java b/giraph-examples/src/test/java/org/apache/giraph/TestBspBasic.java index aa3b08c..38ba27f 100644 --- a/giraph-examples/src/test/java/org/apache/giraph/TestBspBasic.java +++ b/giraph-examples/src/test/java/org/apache/giraph/TestBspBasic.java @@ -18,7 +18,8 @@ package org.apache.giraph; -import org.apache.giraph.conf.GiraphClasses; +import org.apache.giraph.aggregators.TextAggregatorWriter; +import org.apache.giraph.combiner.SimpleSumCombiner; import org.apache.giraph.conf.GiraphConfiguration; import org.apache.giraph.conf.GiraphConstants; import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration; @@ -30,23 +31,21 @@ import org.apache.giraph.examples.SimpleMsgVertex; import org.apache.giraph.examples.SimplePageRankVertex; import org.apache.giraph.examples.SimplePageRankVertex.SimplePageRankVertexInputFormat; import org.apache.giraph.examples.SimpleShortestPathsVertex; -import org.apache.giraph.combiner.SimpleSumCombiner; import org.apache.giraph.examples.SimpleSuperstepVertex; import org.apache.giraph.examples.SimpleSuperstepVertex.SimpleSuperstepVertexInputFormat; import org.apache.giraph.examples.SimpleSuperstepVertex.SimpleSuperstepVertexOutputFormat; import org.apache.giraph.graph.Vertex; -import org.apache.giraph.job.GiraphJob; -import org.apache.giraph.worker.InputSplitPathOrganizer; -import org.apache.giraph.aggregators.TextAggregatorWriter; import org.apache.giraph.io.VertexInputFormat; import org.apache.giraph.io.formats.JsonLongDoubleFloatDoubleVertexOutputFormat; +import org.apache.giraph.job.GiraphJob; +import org.apache.giraph.job.HadoopUtils; +import org.apache.giraph.worker.InputSplitPathOrganizer; import org.apache.giraph.zk.ZooKeeperExt; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.DoubleWritable; import org.apache.hadoop.io.FloatWritable; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; @@ -54,7 +53,7 @@ import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapreduce.InputSplit; -import org.apache.hadoop.mapreduce.JobID; +import org.apache.hadoop.mapreduce.JobContext; import org.apache.zookeeper.KeeperException; import org.junit.Test; @@ -81,12 +80,6 @@ import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -/*if[HADOOP_NON_JOBCONTEXT_IS_INTERFACE] -import org.apache.hadoop.mapreduce.JobContext; -else[HADOOP_NON_JOBCONTEXT_IS_INTERFACE]*/ -import org.apache.hadoop.mapreduce.task.JobContextImpl; -/*end[HADOOP_NON_JOBCONTEXT_IS_INTERFACE]*/ - /** * Unit test for many simple BSP applications. */ @@ -130,15 +123,8 @@ public class System.out.println("testInstantiateVertex: Got vertex " + vertex); VertexInputFormat<LongWritable, IntWritable, FloatWritable> inputFormat = configuration.createWrappedVertexInputFormat(); -/*if[HADOOP_NON_JOBCONTEXT_IS_INTERFACE] - List<InputSplit> splitArray = - inputFormat.getSplits( - new JobContext(new Configuration(), new JobID()), 1); -else[HADOOP_NON_JOBCONTEXT_IS_INTERFACE]*/ - List<InputSplit> splitArray = - inputFormat.getSplits( - new JobContextImpl(new Configuration(), new JobID()), 1); -/*end[HADOOP_NON_JOBCONTEXT_IS_INTERFACE]*/ + List<InputSplit> splitArray = inputFormat.getSplits( + HadoopUtils.makeJobContext(), 1); ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); http://git-wip-us.apache.org/repos/asf/giraph/blob/fd9690a0/giraph-hive/src/test/java/org/apache/giraph/hive/output/HiveOutputTest.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/output/HiveOutputTest.java b/giraph-hive/src/test/java/org/apache/giraph/hive/output/HiveOutputTest.java index 5054728..673c9d5 100644 --- a/giraph-hive/src/test/java/org/apache/giraph/hive/output/HiveOutputTest.java +++ b/giraph-hive/src/test/java/org/apache/giraph/hive/output/HiveOutputTest.java @@ -28,11 +28,12 @@ import org.apache.giraph.io.formats.IntNullTextEdgeInputFormat; import org.apache.giraph.io.internal.WrappedVertexOutputFormat; import org.apache.giraph.utils.InternalVertexRunner; import org.apache.hadoop.mapred.HackJobContext; +import org.apache.hadoop.mapred.HackTaskAttemptContext; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.TaskAttemptID; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.OutputCommitter; import org.apache.hadoop.mapreduce.TaskAttemptContext; -import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.thrift.TException; import org.junit.Before; import org.junit.BeforeClass; @@ -124,11 +125,11 @@ public class HiveOutputTest { throws IOException, InterruptedException { ImmutableClassesGiraphConfiguration iconf = new ImmutableClassesGiraphConfiguration(conf); WrappedVertexOutputFormat outputFormat = iconf.createWrappedVertexOutputFormat(); - TaskAttemptID taskID = new TaskAttemptID(); - TaskAttemptContext taskContext = new TaskAttemptContext(new JobConf(conf), taskID); - OutputCommitter outputCommitter = outputFormat.getOutputCommitter(taskContext); JobConf jobConf = new JobConf(conf); - JobContext jobContext = new HackJobContext(jobConf, taskID.getJobID()); + TaskAttemptContext taskContext = new HackTaskAttemptContext(jobConf, new TaskAttemptID()); + OutputCommitter outputCommitter = outputFormat.getOutputCommitter( + taskContext); + JobContext jobContext = new HackJobContext(jobConf, taskContext.getJobID()); outputCommitter.commitJob(jobContext); } http://git-wip-us.apache.org/repos/asf/giraph/blob/fd9690a0/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 3971076..67dc913 100644 --- a/pom.xml +++ b/pom.xml @@ -254,7 +254,7 @@ under the License. <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <dep.hiveio.version>0.9</dep.hiveio.version> + <dep.hiveio.version>0.10</dep.hiveio.version> <hbase.version>0.90.5</hbase.version> <codehaus-jackson.version>1.8.0</codehaus-jackson.version> <fasterxml-jackson.version>2.1.0</fasterxml-jackson.version> @@ -596,6 +596,7 @@ under the License. <module>giraph-accumulo</module> <module>giraph-hbase</module> <module>giraph-hcatalog</module> + <module>giraph-hive</module> </modules> <activation> <activeByDefault>true</activeByDefault> @@ -625,6 +626,7 @@ under the License. <module>giraph-accumulo</module> <module>giraph-hbase</module> <module>giraph-hcatalog</module> + <module>giraph-hive</module> </modules> <properties> <hadoop.version>1.0.2</hadoop.version> @@ -651,6 +653,7 @@ under the License. <module>giraph-accumulo</module> <module>giraph-hbase</module> <module>giraph-hcatalog</module> + <module>giraph-hive</module> </modules> <properties> <hadoop.version>0.20.2</hadoop.version> @@ -677,6 +680,7 @@ under the License. <module>giraph-accumulo</module> <module>giraph-hbase</module> <module>giraph-hcatalog</module> + <module>giraph-hive</module> </modules> <properties> <hadoop.version>0.20.0</hadoop.version> @@ -1232,7 +1236,6 @@ under the License. <modules> <module>giraph-core</module> - <module>giraph-hive</module> <module>giraph-examples</module> </modules>
