Initial commit of Scrunch test refactoring to support CI Signed-off-by: jwills <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/5b9d3ce1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/5b9d3ce1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/5b9d3ce1 Branch: refs/heads/master Commit: 5b9d3ce1f48377ebc9de0369750a0706b53c1304 Parents: e540459 Author: jwills <[email protected]> Authored: Thu Jul 26 18:33:39 2012 -0700 Committer: jwills <[email protected]> Committed: Sat Jul 28 11:17:58 2012 -0700 ---------------------------------------------------------------------- crunch/pom.xml | 11 +++++ .../java/org/apache/crunch/test/TemporaryPath.java | 5 ++- pom.xml | 7 +++ scrunch/pom.xml | 7 +++ .../org/apache/scrunch/ScrunchTestSupport.java | 32 +++++++++++++++ .../it/scala/org/apache/scrunch/CogroupTest.scala | 9 ++-- .../src/it/scala/org/apache/scrunch/JoinTest.scala | 12 ++--- .../org/apache/scrunch/PageRankClassTest.scala | 10 ++-- .../it/scala/org/apache/scrunch/PageRankTest.scala | 9 ++-- .../scala/org/apache/scrunch/PipelineAppTest.scala | 11 ++--- .../src/it/scala/org/apache/scrunch/TopTest.scala | 7 +-- .../it/scala/org/apache/scrunch/UnionTest.scala | 9 ++-- .../scala/org/apache/scrunch/WordCountTest.scala | 14 ++---- 13 files changed, 96 insertions(+), 47 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5b9d3ce1/crunch/pom.xml ---------------------------------------------------------------------- diff --git a/crunch/pom.xml b/crunch/pom.xml index 0d2f111..a440f2c 100644 --- a/crunch/pom.xml +++ b/crunch/pom.xml @@ -211,6 +211,17 @@ under the License. </execution> </executions> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <executions> + <execution> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> </plugins> </build> http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5b9d3ce1/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java b/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java index dd4f505..e96e3b1 100644 --- a/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java +++ b/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java @@ -6,7 +6,6 @@ import java.io.IOException; import org.apache.crunch.impl.mr.run.RuntimeParameters; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.filter.Filter.ReturnCode; import org.junit.rules.ExternalResource; import org.junit.rules.TemporaryFolder; import org.junit.runner.Description; @@ -102,6 +101,10 @@ public final class TemporaryPath extends ExternalResource { return new Path(file.getAbsolutePath()); } + public Configuration getDefaultConfiguration() throws IOException { + return setTempLoc(new Configuration()); + } + public Configuration setTempLoc(Configuration config) throws IOException { config.set(RuntimeParameters.TMP_DIR, getRootFileName()); config.set("hadoop.tmp.dir", getFileName("hadoop-tmp")); http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5b9d3ce1/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index c106f55..afa73f4 100644 --- a/pom.xml +++ b/pom.xml @@ -106,6 +106,13 @@ under the License. </dependency> <dependency> + <groupId>org.apache.crunch</groupId> + <artifactId>crunch</artifactId> + <type>test-jar</type> + <version>${project.version}</version> + </dependency> + + <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <version>11.0.2</version> http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5b9d3ce1/scrunch/pom.xml ---------------------------------------------------------------------- diff --git a/scrunch/pom.xml b/scrunch/pom.xml index d70b1d2..0b549ae 100644 --- a/scrunch/pom.xml +++ b/scrunch/pom.xml @@ -70,6 +70,12 @@ under the License. <artifactId>scalatest_${scala.version}</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.crunch</groupId> + <artifactId>crunch</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> </dependencies> <build> @@ -152,6 +158,7 @@ under the License. </goals> <configuration> <sources> + <source>${basedir}/src/it/java</source> <source>${basedir}/src/it/scala</source> </sources> </configuration> http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5b9d3ce1/scrunch/src/it/java/org/apache/scrunch/ScrunchTestSupport.java ---------------------------------------------------------------------- diff --git a/scrunch/src/it/java/org/apache/scrunch/ScrunchTestSupport.java b/scrunch/src/it/java/org/apache/scrunch/ScrunchTestSupport.java new file mode 100644 index 0000000..2cfb1ce --- /dev/null +++ b/scrunch/src/it/java/org/apache/scrunch/ScrunchTestSupport.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.scrunch; + +import org.apache.crunch.test.TemporaryPath; + +import org.junit.Rule; +import org.scalatest.junit.JUnitSuite; + +public class ScrunchTestSupport { + @Rule + private final TemporaryPath tempDir = new TemporaryPath(); + + public TemporaryPath tempDir() { + return tempDir; + } +} http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5b9d3ce1/scrunch/src/it/scala/org/apache/scrunch/CogroupTest.scala ---------------------------------------------------------------------- diff --git a/scrunch/src/it/scala/org/apache/scrunch/CogroupTest.scala b/scrunch/src/it/scala/org/apache/scrunch/CogroupTest.scala index b0e94a1..268d2b6 100644 --- a/scrunch/src/it/scala/org/apache/scrunch/CogroupTest.scala +++ b/scrunch/src/it/scala/org/apache/scrunch/CogroupTest.scala @@ -18,13 +18,12 @@ package org.apache.scrunch import org.apache.crunch.io.{From => from} -import org.apache.crunch.test.FileHelper import org.scalatest.junit.JUnitSuite import _root_.org.junit.Test -class CogroupTest extends JUnitSuite { - val pipeline = Pipeline.mapReduce[CogroupTest] +class CogroupTest extends ScrunchTestSupport with JUnitSuite { + val pipeline = Pipeline.mapReduce[CogroupTest](tempDir.getDefaultConfiguration) def wordCount(fileName: String) = { pipeline.read(from.textFile(fileName)) @@ -32,8 +31,8 @@ class CogroupTest extends JUnitSuite { } @Test def cogroup { - val shakespeare = FileHelper.createTempCopyOf("shakes.txt") - val maugham = FileHelper.createTempCopyOf("maugham.txt") + val shakespeare = tempDir.copyResourceFileName("shakes.txt") + val maugham = tempDir.copyResourceFileName("maugham.txt") val diffs = wordCount(shakespeare).cogroup(wordCount(maugham)) .map((k, v) => (k, (v._1.sum - v._2.sum))).materialize assert(diffs.exists(_ == ("the", -11390))) http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5b9d3ce1/scrunch/src/it/scala/org/apache/scrunch/JoinTest.scala ---------------------------------------------------------------------- diff --git a/scrunch/src/it/scala/org/apache/scrunch/JoinTest.scala b/scrunch/src/it/scala/org/apache/scrunch/JoinTest.scala index 5303c03..c3a4a80 100644 --- a/scrunch/src/it/scala/org/apache/scrunch/JoinTest.scala +++ b/scrunch/src/it/scala/org/apache/scrunch/JoinTest.scala @@ -18,13 +18,12 @@ package org.apache.scrunch import org.apache.crunch.io.{From => from, To => to} -import org.apache.crunch.test.FileHelper import org.scalatest.junit.JUnitSuite import _root_.org.junit.Test -class JoinTest extends JUnitSuite { - val pipeline = Pipeline.mapReduce[CogroupTest] +class JoinTest extends ScrunchTestSupport with JUnitSuite { + val pipeline = Pipeline.mapReduce[CogroupTest](tempDir.getDefaultConfiguration) def wordCount(fileName: String) = { pipeline.read(from.textFile(fileName)) @@ -32,10 +31,9 @@ class JoinTest extends JUnitSuite { } @Test def join { - val shakespeare = FileHelper.createTempCopyOf("shakes.txt") - val maugham = FileHelper.createTempCopyOf("maugham.txt") - val output = FileHelper.createOutputPath() - output.deleteOnExit() + val shakespeare = tempDir.copyResourceFileName("shakes.txt") + val maugham = tempDir.copyResourceFileName("maugham.txt") + val output = tempDir.getFile("output") val filtered = wordCount(shakespeare).join(wordCount(maugham)) .map((k, v) => (k, v._1 - v._2)) .write(to.textFile(output.getAbsolutePath())) http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5b9d3ce1/scrunch/src/it/scala/org/apache/scrunch/PageRankClassTest.scala ---------------------------------------------------------------------- diff --git a/scrunch/src/it/scala/org/apache/scrunch/PageRankClassTest.scala b/scrunch/src/it/scala/org/apache/scrunch/PageRankClassTest.scala index 9ab7897..43775ea 100644 --- a/scrunch/src/it/scala/org/apache/scrunch/PageRankClassTest.scala +++ b/scrunch/src/it/scala/org/apache/scrunch/PageRankClassTest.scala @@ -21,7 +21,6 @@ import Avros._ import org.apache.crunch.{DoFn, Emitter, Pair => P} import org.apache.crunch.io.{From => from} -import org.apache.crunch.test.FileHelper import scala.collection.mutable.HashMap @@ -61,8 +60,9 @@ class CachingPageRankClassFn extends DoFn[P[String, PageRankData], P[String, Flo } } -class PageRankClassTest extends JUnitSuite { - val pipeline = Pipeline.mapReduce[PageRankTest] +class PageRankClassTest extends ScrunchTestSupport with JUnitSuite { + + val pipeline = Pipeline.mapReduce[PageRankTest](tempDir.getDefaultConfiguration) def initialInput(fileName: String) = { pipeline.read(from.textFile(fileName)) @@ -94,7 +94,7 @@ class PageRankClassTest extends JUnitSuite { @Test def testPageRank { pipeline.getConfiguration.set("crunch.debug", "true") - var prev = initialInput(FileHelper.createTempCopyOf("urls.txt")) + var prev = initialInput(tempDir.copyResourceFileName("urls.txt")) var delta = 1.0f while (delta > 0.01f) { prev = update(prev, 0.5f) @@ -106,7 +106,7 @@ class PageRankClassTest extends JUnitSuite { def testFastPageRank { pipeline.getConfiguration.set("crunch.debug", "true") - var prev = initialInput(FileHelper.createTempCopyOf("urls.txt")) + var prev = initialInput(tempDir.copyResourceFileName("urls.txt")) var delta = 1.0f while (delta > 0.01f) { prev = fastUpdate(prev, 0.5f) http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5b9d3ce1/scrunch/src/it/scala/org/apache/scrunch/PageRankTest.scala ---------------------------------------------------------------------- diff --git a/scrunch/src/it/scala/org/apache/scrunch/PageRankTest.scala b/scrunch/src/it/scala/org/apache/scrunch/PageRankTest.scala index cbf7ebf..3cf7b8a 100644 --- a/scrunch/src/it/scala/org/apache/scrunch/PageRankTest.scala +++ b/scrunch/src/it/scala/org/apache/scrunch/PageRankTest.scala @@ -21,7 +21,6 @@ import Avros._ import org.apache.crunch.{DoFn, Emitter, Pair => P} import org.apache.crunch.io.{From => from} -import org.apache.crunch.test.FileHelper import scala.collection.mutable.HashMap @@ -49,8 +48,8 @@ class CachingPageRankFn extends DoFn[P[String, (Float, Float, List[String])], P[ } } -class PageRankTest extends JUnitSuite { - val pipeline = Pipeline.mapReduce[PageRankTest] +class PageRankTest extends ScrunchTestSupport with JUnitSuite { + val pipeline = Pipeline.mapReduce[PageRankTest](tempDir.getDefaultConfiguration) def initialInput(fileName: String) = { pipeline.read(from.textFile(fileName)) @@ -82,7 +81,7 @@ class PageRankTest extends JUnitSuite { } @Test def testPageRank { - var prev = initialInput(FileHelper.createTempCopyOf("urls.txt")) + var prev = initialInput(tempDir.copyResourceFileName("urls.txt")) var delta = 1.0f while (delta > 0.01f) { prev = update(prev, 0.5f) @@ -93,7 +92,7 @@ class PageRankTest extends JUnitSuite { } @Test def testFastPageRank { - var prev = initialInput(FileHelper.createTempCopyOf("urls.txt")) + var prev = initialInput(tempDir.copyResourceFileName("urls.txt")) var delta = 1.0f while (delta > 0.01f) { prev = fastUpdate(prev, 0.5f) http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5b9d3ce1/scrunch/src/it/scala/org/apache/scrunch/PipelineAppTest.scala ---------------------------------------------------------------------- diff --git a/scrunch/src/it/scala/org/apache/scrunch/PipelineAppTest.scala b/scrunch/src/it/scala/org/apache/scrunch/PipelineAppTest.scala index 7111c7b..c61e7d1 100644 --- a/scrunch/src/it/scala/org/apache/scrunch/PipelineAppTest.scala +++ b/scrunch/src/it/scala/org/apache/scrunch/PipelineAppTest.scala @@ -15,8 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import org.apache.crunch.test.FileHelper -import org.apache.scrunch.PipelineApp +package org.apache.scrunch import org.scalatest.junit.JUnitSuite import _root_.org.junit.Test @@ -36,12 +35,12 @@ object WordCount extends PipelineApp { cogroup(w1, w2).write(to.textFile(args(2))) } -class PipelineAppTest extends JUnitSuite { +class PipelineAppTest extends ScrunchTestSupport with JUnitSuite { @Test def run { val args = new Array[String](3) - args(0) = FileHelper.createTempCopyOf("shakes.txt") - args(1) = FileHelper.createTempCopyOf("maugham.txt") - args(2) = FileHelper.createOutputPath.getAbsolutePath + args(0) = tempDir.copyResourceFileName("shakes.txt") + args(1) = tempDir.copyResourceFileName("maugham.txt") + args(2) = tempDir.getFileName("output") WordCount.main(args) } } http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5b9d3ce1/scrunch/src/it/scala/org/apache/scrunch/TopTest.scala ---------------------------------------------------------------------- diff --git a/scrunch/src/it/scala/org/apache/scrunch/TopTest.scala b/scrunch/src/it/scala/org/apache/scrunch/TopTest.scala index bac27bd..9c40b5c 100644 --- a/scrunch/src/it/scala/org/apache/scrunch/TopTest.scala +++ b/scrunch/src/it/scala/org/apache/scrunch/TopTest.scala @@ -18,12 +18,11 @@ package org.apache.scrunch import org.apache.crunch.io.{From => from, To => to} -import org.apache.crunch.test.FileHelper import org.scalatest.junit.JUnitSuite import _root_.org.junit.Test -class TopTest extends JUnitSuite { +class TopTest extends ScrunchTestSupport with JUnitSuite { @Test def topInMem { val ptable = Mem.tableOf(("foo", 17), ("bar", 29), ("baz", 1729)) @@ -31,8 +30,8 @@ class TopTest extends JUnitSuite { } @Test def top2 { - val pipeline = Pipeline.mapReduce[TopTest] - val input = FileHelper.createTempCopyOf("shakes.txt") + val pipeline = Pipeline.mapReduce[TopTest](tempDir.getDefaultConfiguration) + val input = tempDir.copyResourceFileName("shakes.txt") val wc = pipeline.read(from.textFile(input)) .flatMap(_.toLowerCase.split("\\s+")) http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5b9d3ce1/scrunch/src/it/scala/org/apache/scrunch/UnionTest.scala ---------------------------------------------------------------------- diff --git a/scrunch/src/it/scala/org/apache/scrunch/UnionTest.scala b/scrunch/src/it/scala/org/apache/scrunch/UnionTest.scala index 63fecdb..fa88edc 100644 --- a/scrunch/src/it/scala/org/apache/scrunch/UnionTest.scala +++ b/scrunch/src/it/scala/org/apache/scrunch/UnionTest.scala @@ -18,15 +18,14 @@ package org.apache.scrunch import org.apache.crunch.io.{From => from} -import org.apache.crunch.test.FileHelper import org.scalatest.junit.JUnitSuite import _root_.org.junit.Test -class UnionTest extends JUnitSuite { - val pipeline = Pipeline.mapReduce[UnionTest] - val shakespeare = FileHelper.createTempCopyOf("shakes.txt") - val maugham = FileHelper.createTempCopyOf("maugham.txt") +class UnionTest extends ScrunchTestSupport with JUnitSuite { + val pipeline = Pipeline.mapReduce[UnionTest](tempDir.getDefaultConfiguration) + val shakespeare = tempDir.copyResourceFileName("shakes.txt") + val maugham = tempDir.copyResourceFileName("maugham.txt") def wordCount(col: PCollection[String]) = { col.flatMap(_.toLowerCase.split("\\W+")).count http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5b9d3ce1/scrunch/src/it/scala/org/apache/scrunch/WordCountTest.scala ---------------------------------------------------------------------- diff --git a/scrunch/src/it/scala/org/apache/scrunch/WordCountTest.scala b/scrunch/src/it/scala/org/apache/scrunch/WordCountTest.scala index e97a1fd..f8081ea 100644 --- a/scrunch/src/it/scala/org/apache/scrunch/WordCountTest.scala +++ b/scrunch/src/it/scala/org/apache/scrunch/WordCountTest.scala @@ -18,28 +18,24 @@ package org.apache.scrunch import org.apache.crunch.io.{From => from, To => to} -import org.apache.crunch.test.FileHelper - -import java.io.File import org.scalatest.junit.JUnitSuite import _root_.org.junit.Test -class WordCountTest extends JUnitSuite { +class WordCountTest extends ScrunchTestSupport with JUnitSuite { @Test def wordCount { - val pipeline = Pipeline.mapReduce[WordCountTest] - val input = FileHelper.createTempCopyOf("shakes.txt") - val wordCountOut = FileHelper.createOutputPath + val pipeline = Pipeline.mapReduce[WordCountTest](tempDir.getDefaultConfiguration) + val input = tempDir.copyResourceFileName("shakes.txt") + val wordCountOut = tempDir.getFileName("output") val fcc = pipeline.read(from.textFile(input)) .flatMap(_.toLowerCase.split("\\s+")) .filter(!_.isEmpty()).count - .write(to.textFile(wordCountOut.getAbsolutePath)) // Word counts + .write(to.textFile(wordCountOut)) // Word counts .map((w, c) => (w.slice(0, 1), c)) .groupByKey.combine(v => v.sum).materialize assert(fcc.exists(_ == ("w", 1404))) pipeline.done - wordCountOut.delete() } }
