CRUNCH-24: Clean up test suite output. Add new junit @Rule TemporaryPath that provides a temporary directory. The directory and all files inside are deleted after a test run. Change integration tests to use TemporaryPath where necessary.
Signed-off-by: jwills <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/6f549ed3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/6f549ed3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/6f549ed3 Branch: refs/heads/master Commit: 6f549ed3ed0267b0872b5d3256a6f313a1a412c8 Parents: b645a55 Author: Matthias Friedrich <[email protected]> Authored: Sun Jul 22 09:27:15 2012 +0200 Committer: jwills <[email protected]> Committed: Wed Jul 25 22:23:37 2012 -0700 ---------------------------------------------------------------------- .../java/org/apache/crunch/MultipleOutputIT.java | 16 +-- .../org/apache/crunch/PCollectionGetSizeIT.java | 11 +- .../it/java/org/apache/crunch/TermFrequencyIT.java | 11 +- crunch/src/it/java/org/apache/crunch/TfIdfIT.java | 33 ++--- .../org/apache/crunch/TupleNClassCastBugIT.java | 21 ++-- .../java/org/apache/crunch/WordCountHBaseIT.java | 14 +-- .../src/it/java/org/apache/crunch/WordCountIT.java | 19 ++-- .../crunch/impl/mr/collect/UnionCollectionIT.java | 35 +++--- .../it/java/org/apache/crunch/lib/CogroupIT.java | 13 +- .../java/org/apache/crunch/test/TemporaryPath.java | 105 +++++++++++++++ .../org/apache/crunch/test/TemporaryPathIT.java | 59 ++++++++ 11 files changed, 244 insertions(+), 93 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/6f549ed3/crunch/src/it/java/org/apache/crunch/MultipleOutputIT.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/MultipleOutputIT.java b/crunch/src/it/java/org/apache/crunch/MultipleOutputIT.java index 7f50b54..90925ba 100644 --- a/crunch/src/it/java/org/apache/crunch/MultipleOutputIT.java +++ b/crunch/src/it/java/org/apache/crunch/MultipleOutputIT.java @@ -27,15 +27,18 @@ import java.util.List; import org.apache.crunch.impl.mr.MRPipeline; import org.apache.crunch.io.At; -import org.apache.crunch.test.FileHelper; +import org.apache.crunch.test.TemporaryPath; import org.apache.crunch.types.PTypeFamily; import org.apache.crunch.types.avro.AvroTypeFamily; import org.apache.crunch.types.writable.WritableTypeFamily; +import org.junit.Rule; import org.junit.Test; import com.google.common.io.Files; public class MultipleOutputIT { + @Rule + public TemporaryPath tmpDir = new TemporaryPath(); public static PCollection<String> evenCountLetters(PCollection<String> words, PTypeFamily typeFamily) { return words.parallelDo("even", new FilterFn<String>() { @@ -79,11 +82,9 @@ public class MultipleOutputIT { } public void run(Pipeline pipeline, PTypeFamily typeFamily) throws IOException { - String inputPath = FileHelper.createTempCopyOf("letters.txt"); - File outputEven = FileHelper.createOutputPath(); - File outputOdd = FileHelper.createOutputPath(); - String outputPathEven = outputEven.getAbsolutePath(); - String outputPathOdd = outputOdd.getAbsolutePath(); + String inputPath = tmpDir.copyResourceFileName("letters.txt"); + String outputPathEven = tmpDir.getFileName("even"); + String outputPathOdd = tmpDir.getFileName("odd"); PCollection<String> words = pipeline.read(At.textFile(inputPath, typeFamily.strings())); @@ -96,9 +97,6 @@ public class MultipleOutputIT { checkFileContents(outputPathEven, Arrays.asList("bb")); checkFileContents(outputPathOdd, Arrays.asList("a")); - - outputEven.deleteOnExit(); - outputOdd.deleteOnExit(); } private void checkFileContents(String filePath, List<String> expected) throws IOException { http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/6f549ed3/crunch/src/it/java/org/apache/crunch/PCollectionGetSizeIT.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/PCollectionGetSizeIT.java b/crunch/src/it/java/org/apache/crunch/PCollectionGetSizeIT.java index a6571bb..4ff8490 100644 --- a/crunch/src/it/java/org/apache/crunch/PCollectionGetSizeIT.java +++ b/crunch/src/it/java/org/apache/crunch/PCollectionGetSizeIT.java @@ -28,12 +28,15 @@ import java.io.IOException; import org.apache.crunch.impl.mem.MemPipeline; import org.apache.crunch.impl.mr.MRPipeline; -import org.apache.crunch.test.FileHelper; +import org.apache.crunch.test.TemporaryPath; import org.junit.Before; import org.junit.Ignore; +import org.junit.Rule; import org.junit.Test; public class PCollectionGetSizeIT { + @Rule + public TemporaryPath tmpDir = new TemporaryPath(); private String emptyInputPath; private String nonEmptyInputPath; @@ -51,9 +54,9 @@ public class PCollectionGetSizeIT { @Before public void setUp() throws IOException { - emptyInputPath = FileHelper.createTempCopyOf("emptyTextFile.txt"); - nonEmptyInputPath = FileHelper.createTempCopyOf("set1.txt"); - outputPath = FileHelper.createOutputPath().getAbsolutePath(); + emptyInputPath = tmpDir.copyResourceFileName("emptyTextFile.txt"); + nonEmptyInputPath = tmpDir.copyResourceFileName("set1.txt"); + outputPath = tmpDir.getFileName("output"); } @Test http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/6f549ed3/crunch/src/it/java/org/apache/crunch/TermFrequencyIT.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/TermFrequencyIT.java b/crunch/src/it/java/org/apache/crunch/TermFrequencyIT.java index e0d0b47..ee2267d 100644 --- a/crunch/src/it/java/org/apache/crunch/TermFrequencyIT.java +++ b/crunch/src/it/java/org/apache/crunch/TermFrequencyIT.java @@ -29,13 +29,16 @@ import org.apache.crunch.impl.mr.MRPipeline; import org.apache.crunch.io.At; import org.apache.crunch.io.ReadableSourceTarget; import org.apache.crunch.lib.Aggregate; -import org.apache.crunch.test.FileHelper; +import org.apache.crunch.test.TemporaryPath; import org.apache.crunch.types.PTypeFamily; import org.apache.crunch.types.writable.WritableTypeFamily; +import org.junit.Rule; import org.junit.Test; @SuppressWarnings("serial") public class TermFrequencyIT implements Serializable { + @Rule + public transient TemporaryPath tmpDir = new TemporaryPath(); @Test public void testTermFrequencyWithNoTransform() throws IOException { @@ -58,10 +61,10 @@ public class TermFrequencyIT implements Serializable { } public void run(Pipeline pipeline, PTypeFamily typeFamily, boolean transformTF) throws IOException { - String input = FileHelper.createTempCopyOf("docs.txt"); + String input = tmpDir.copyResourceFileName("docs.txt"); - File transformedOutput = FileHelper.createOutputPath(); - File tfOutput = FileHelper.createOutputPath(); + File transformedOutput = tmpDir.getFile("transformed-output"); + File tfOutput = tmpDir.getFile("tf-output"); PCollection<String> docs = pipeline.readTextFile(input); http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/6f549ed3/crunch/src/it/java/org/apache/crunch/TfIdfIT.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/TfIdfIT.java b/crunch/src/it/java/org/apache/crunch/TfIdfIT.java index f48de8c..de1d5dc 100644 --- a/crunch/src/it/java/org/apache/crunch/TfIdfIT.java +++ b/crunch/src/it/java/org/apache/crunch/TfIdfIT.java @@ -17,8 +17,6 @@ */ package org.apache.crunch; -import static com.google.common.io.Resources.getResource; -import static com.google.common.io.Resources.newInputStreamSupplier; import static org.junit.Assert.assertTrue; import java.io.File; @@ -33,9 +31,11 @@ import org.apache.crunch.impl.mr.MRPipeline; import org.apache.crunch.io.seq.SeqFileSourceTarget; import org.apache.crunch.lib.Aggregate; import org.apache.crunch.lib.Join; +import org.apache.crunch.test.TemporaryPath; import org.apache.crunch.types.PTypeFamily; import org.apache.crunch.types.writable.WritableTypeFamily; import org.apache.hadoop.fs.Path; +import org.junit.Rule; import org.junit.Test; import com.google.common.collect.Lists; @@ -43,6 +43,9 @@ import com.google.common.io.Files; @SuppressWarnings("serial") public class TfIdfIT implements Serializable { + @Rule + public transient TemporaryPath tmpDir = new TemporaryPath(); + // total number of documents, should calculate protected static final double N = 2; @@ -169,16 +172,13 @@ public class TfIdfIT implements Serializable { } public void run(Pipeline pipeline, PTypeFamily typeFamily, boolean singleRun) throws IOException { - File input = File.createTempFile("docs", "txt"); - input.deleteOnExit(); - Files.copy(newInputStreamSupplier(getResource("docs.txt")), input); - - String outputPath1 = getOutput(); - String outputPath2 = getOutput(); + String inputFile = tmpDir.copyResourceFileName("docs.txt"); + String outputPath1 = tmpDir.getFileName("output1"); + String outputPath2 = tmpDir.getFileName("output2"); - Path tfPath = new Path(getOutput("termfreq")); + Path tfPath = tmpDir.getPath("termfreq"); - PCollection<String> docs = pipeline.readTextFile(input.getAbsolutePath()); + PCollection<String> docs = pipeline.readTextFile(inputFile); PTable<String, Collection<Pair<String, Double>>> results = generateTFIDF(docs, tfPath, typeFamily); pipeline.writeTextFile(results, outputPath1); @@ -198,7 +198,6 @@ public class TfIdfIT implements Serializable { // Check the lowercase version... File outputFile = new File(outputPath1, "part-r-00000"); - outputFile.deleteOnExit(); List<String> lines = Files.readLines(outputFile, Charset.defaultCharset()); boolean passed = false; for (String line : lines) { @@ -211,7 +210,6 @@ public class TfIdfIT implements Serializable { // ...and the uppercase version outputFile = new File(outputPath2, "part-r-00000"); - outputFile.deleteOnExit(); lines = Files.readLines(outputFile, Charset.defaultCharset()); passed = false; for (String line : lines) { @@ -222,15 +220,4 @@ public class TfIdfIT implements Serializable { } assertTrue(passed); } - - public static String getOutput() throws IOException { - return getOutput("output"); - } - - public static String getOutput(String prefix) throws IOException { - File output = File.createTempFile(prefix, ""); - String path = output.getAbsolutePath(); - output.delete(); - return path; - } } http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/6f549ed3/crunch/src/it/java/org/apache/crunch/TupleNClassCastBugIT.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/TupleNClassCastBugIT.java b/crunch/src/it/java/org/apache/crunch/TupleNClassCastBugIT.java index 39919ba..f845214 100644 --- a/crunch/src/it/java/org/apache/crunch/TupleNClassCastBugIT.java +++ b/crunch/src/it/java/org/apache/crunch/TupleNClassCastBugIT.java @@ -17,8 +17,6 @@ */ package org.apache.crunch; -import static com.google.common.io.Resources.getResource; -import static com.google.common.io.Resources.newInputStreamSupplier; import static org.junit.Assert.assertEquals; import java.io.File; @@ -27,14 +25,19 @@ import java.nio.charset.Charset; import java.util.List; import org.apache.crunch.impl.mr.MRPipeline; +import org.apache.crunch.test.TemporaryPath; import org.apache.crunch.types.PTypeFamily; import org.apache.crunch.types.avro.AvroTypeFamily; import org.apache.crunch.types.writable.WritableTypeFamily; +import org.junit.Rule; import org.junit.Test; import com.google.common.io.Files; + public class TupleNClassCastBugIT { + @Rule + public TemporaryPath tmpDir = new TemporaryPath(); public static PCollection<TupleN> mapGroupDo(PCollection<String> lines, PTypeFamily ptf) { PTable<String, TupleN> mapped = lines.parallelDo(new MapFn<String, Pair<String, TupleN>>() { @@ -68,15 +71,10 @@ public class TupleNClassCastBugIT { } public void run(Pipeline pipeline, PTypeFamily typeFamily) throws IOException { - File input = File.createTempFile("docs", "txt"); - input.deleteOnExit(); - Files.copy(newInputStreamSupplier(getResource("docs.txt")), input); - - File output = File.createTempFile("output", ""); - String outputPath = output.getAbsolutePath(); - output.delete(); + String inputPath = tmpDir.copyResourceFileName("docs.txt"); + String outputPath = tmpDir.getFileName("output"); - PCollection<String> docLines = pipeline.readTextFile(input.getAbsolutePath()); + PCollection<String> docLines = pipeline.readTextFile(inputPath); pipeline.writeTextFile(mapGroupDo(docLines, typeFamily), outputPath); pipeline.done(); @@ -85,13 +83,12 @@ public class TupleNClassCastBugIT { // *** which is thrown in a different thread during the reduce phase. If all // is well // *** the file will exist and have six lines. Otherwise the bug is present. - File outputFile = new File(output, "part-r-00000"); + File outputFile = new File(outputPath, "part-r-00000"); List<String> lines = Files.readLines(outputFile, Charset.defaultCharset()); int lineCount = 0; for (String line : lines) { lineCount++; } assertEquals(6, lineCount); - output.deleteOnExit(); } } http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/6f549ed3/crunch/src/it/java/org/apache/crunch/WordCountHBaseIT.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/WordCountHBaseIT.java b/crunch/src/it/java/org/apache/crunch/WordCountHBaseIT.java index 9ff171d..9704c3b 100644 --- a/crunch/src/it/java/org/apache/crunch/WordCountHBaseIT.java +++ b/crunch/src/it/java/org/apache/crunch/WordCountHBaseIT.java @@ -28,12 +28,11 @@ import java.util.Random; import java.util.jar.JarEntry; import java.util.jar.JarOutputStream; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.crunch.impl.mr.MRPipeline; import org.apache.crunch.io.hbase.HBaseSourceTarget; import org.apache.crunch.io.hbase.HBaseTarget; import org.apache.crunch.lib.Aggregate; +import org.apache.crunch.test.TemporaryPath; import org.apache.crunch.types.writable.Writables; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.filecache.DistributedCache; @@ -52,12 +51,14 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.mapred.TaskAttemptContext; import org.junit.After; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import com.google.common.io.ByteStreams; public class WordCountHBaseIT { - protected static final Log LOG = LogFactory.getLog(WordCountHBaseIT.class); + @Rule + public TemporaryPath tmpDir = new TemporaryPath(); private static final byte[] COUNTS_COLFAM = Bytes.toBytes("cf"); private static final byte[] WORD_COLFAM = Bytes.toBytes("cf"); @@ -88,15 +89,10 @@ public class WordCountHBaseIT { }, Writables.writables(Put.class)); } - @SuppressWarnings("deprecation") @Before public void setUp() throws Exception { Configuration conf = hbaseTestUtil.getConfiguration(); - File tmpDir = File.createTempFile("logdir", ""); - tmpDir.delete(); - tmpDir.mkdir(); - tmpDir.deleteOnExit(); - conf.set("hadoop.log.dir", tmpDir.getAbsolutePath()); + conf.set("hadoop.log.dir", tmpDir.getRootFileName()); conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1"); conf.setInt("hbase.master.info.port", -1); conf.setInt("hbase.regionserver.info.port", -1); http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/6f549ed3/crunch/src/it/java/org/apache/crunch/WordCountIT.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/WordCountIT.java b/crunch/src/it/java/org/apache/crunch/WordCountIT.java index ba457ee..5ba931f 100644 --- a/crunch/src/it/java/org/apache/crunch/WordCountIT.java +++ b/crunch/src/it/java/org/apache/crunch/WordCountIT.java @@ -29,10 +29,11 @@ import org.apache.crunch.impl.mr.MRPipeline; import org.apache.crunch.io.At; import org.apache.crunch.io.To; import org.apache.crunch.lib.Aggregate; -import org.apache.crunch.test.FileHelper; +import org.apache.crunch.test.TemporaryPath; import org.apache.crunch.types.PTypeFamily; import org.apache.crunch.types.avro.AvroTypeFamily; import org.apache.crunch.types.writable.WritableTypeFamily; +import org.junit.Rule; import org.junit.Test; import com.google.common.collect.ImmutableList; @@ -40,6 +41,8 @@ import com.google.common.collect.Lists; import com.google.common.io.Files; public class WordCountIT { + @Rule + public TemporaryPath tmpDir = new TemporaryPath(); enum WordCountStats { ANDS @@ -113,9 +116,9 @@ public class WordCountIT { runWithTop(AvroTypeFamily.getInstance()); } - public static void runWithTop(PTypeFamily tf) throws IOException { + public void runWithTop(PTypeFamily tf) throws IOException { Pipeline pipeline = new MRPipeline(WordCountIT.class); - String inputPath = FileHelper.createTempCopyOf("shakes.txt"); + String inputPath = tmpDir.copyResourceFileName("shakes.txt"); PCollection<String> shakespeare = pipeline.read(At.textFile(inputPath, tf.strings())); PTable<String, Long> wordCount = wordCount(shakespeare, tf); @@ -126,9 +129,8 @@ public class WordCountIT { } public void run(Pipeline pipeline, PTypeFamily typeFamily) throws IOException { - String inputPath = FileHelper.createTempCopyOf("shakes.txt"); - File output = FileHelper.createOutputPath(); - String outputPath = output.getAbsolutePath(); + String inputPath = tmpDir.copyResourceFileName("shakes.txt"); + String outputPath = tmpDir.getFileName("output"); PCollection<String> shakespeare = pipeline.read(At.textFile(inputPath, typeFamily.strings())); PTable<String, Long> wordCount = wordCount(shakespeare, typeFamily); @@ -139,9 +141,7 @@ public class WordCountIT { } if (runSecond) { - File substrCount = File.createTempFile("substr", ""); - String substrPath = substrCount.getAbsolutePath(); - substrCount.delete(); + String substrPath = tmpDir.getFileName("substr"); PTable<String, Long> we = substr(wordCount).groupByKey().combineValues(CombineFn.<String> SUM_LONGS()); pipeline.writeTextFile(we, substrPath); } @@ -165,6 +165,5 @@ public class WordCountIT { } } assertTrue(passed); - output.deleteOnExit(); } } http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/6f549ed3/crunch/src/it/java/org/apache/crunch/impl/mr/collect/UnionCollectionIT.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/impl/mr/collect/UnionCollectionIT.java b/crunch/src/it/java/org/apache/crunch/impl/mr/collect/UnionCollectionIT.java index 1bb3305..9748143 100644 --- a/crunch/src/it/java/org/apache/crunch/impl/mr/collect/UnionCollectionIT.java +++ b/crunch/src/it/java/org/apache/crunch/impl/mr/collect/UnionCollectionIT.java @@ -36,13 +36,14 @@ import org.apache.crunch.impl.mem.MemPipeline; import org.apache.crunch.impl.mr.MRPipeline; import org.apache.crunch.io.At; import org.apache.crunch.io.To; -import org.apache.crunch.test.FileHelper; +import org.apache.crunch.test.TemporaryPath; import org.apache.crunch.types.PTypeFamily; import org.apache.crunch.types.avro.AvroTypeFamily; import org.apache.crunch.types.avro.Avros; import org.apache.crunch.types.writable.WritableTypeFamily; import org.junit.After; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -52,6 +53,8 @@ import com.google.common.collect.Lists; @RunWith(value = Parameterized.class) public class UnionCollectionIT { + @Rule + public TemporaryPath tmpDir = new TemporaryPath(); private static final Log LOG = LogFactory.getLog(UnionCollectionIT.class); @@ -64,8 +67,8 @@ public class UnionCollectionIT { @Before @SuppressWarnings("unchecked") public void setUp() throws IOException { - String inputFile1 = FileHelper.createTempCopyOf("set1.txt"); - String inputFile2 = FileHelper.createTempCopyOf("set2.txt"); + String inputFile1 = tmpDir.copyResourceFileName("set1.txt"); + String inputFile2 = tmpDir.copyResourceFileName("set2.txt"); PCollection<String> firstCollection = pipeline.read(At.textFile(inputFile1, typeFamily.strings())); PCollection<String> secondCollection = pipeline.read(At.textFile(inputFile2, typeFamily.strings())); @@ -114,30 +117,30 @@ public class UnionCollectionIT { @Test public void unionWriteShouldNotThrowNPE() throws IOException { - File outputPath1 = FileHelper.createOutputPath(); - File outputPath2 = FileHelper.createOutputPath(); - File outputPath3 = FileHelper.createOutputPath(); + String outputPath1 = tmpDir.getFileName("output1"); + String outputPath2 = tmpDir.getFileName("output2"); + String outputPath3 = tmpDir.getFileName("output3"); if (typeFamily == AvroTypeFamily.getInstance()) { - union.write(To.avroFile(outputPath1.getAbsolutePath())); - pipeline.write(union, To.avroFile(outputPath2.getAbsolutePath())); + union.write(To.avroFile(outputPath1)); + pipeline.write(union, To.avroFile(outputPath2)); pipeline.run(); - checkFileContents(outputPath1.getAbsolutePath()); - checkFileContents(outputPath2.getAbsolutePath()); + checkFileContents(outputPath1); + checkFileContents(outputPath2); } else { - union.write(To.textFile(outputPath1.getAbsolutePath())); - pipeline.write(union, To.textFile(outputPath2.getAbsolutePath())); - pipeline.writeTextFile(union, outputPath3.getAbsolutePath()); + union.write(To.textFile(outputPath1)); + pipeline.write(union, To.textFile(outputPath2)); + pipeline.writeTextFile(union, outputPath3); pipeline.run(); - checkFileContents(outputPath1.getAbsolutePath()); - checkFileContents(outputPath2.getAbsolutePath()); - checkFileContents(outputPath3.getAbsolutePath()); + checkFileContents(outputPath1); + checkFileContents(outputPath2); + checkFileContents(outputPath3); } } http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/6f549ed3/crunch/src/it/java/org/apache/crunch/lib/CogroupIT.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/lib/CogroupIT.java b/crunch/src/it/java/org/apache/crunch/lib/CogroupIT.java index c64b8ec..4f65964 100644 --- a/crunch/src/it/java/org/apache/crunch/lib/CogroupIT.java +++ b/crunch/src/it/java/org/apache/crunch/lib/CogroupIT.java @@ -36,17 +36,20 @@ import org.apache.crunch.fn.MapKeysFn; import org.apache.crunch.fn.MapValuesFn; import org.apache.crunch.impl.mr.MRPipeline; import org.apache.crunch.io.From; -import org.apache.crunch.test.FileHelper; +import org.apache.crunch.test.TemporaryPath; import org.apache.crunch.types.PTableType; import org.apache.crunch.types.PTypeFamily; import org.apache.crunch.types.avro.AvroTypeFamily; import org.apache.crunch.types.writable.WritableTypeFamily; +import org.junit.Rule; import org.junit.Test; import com.google.common.base.Splitter; import com.google.common.io.Files; public class CogroupIT { + @Rule + public TemporaryPath tmpDir = new TemporaryPath(); private static class WordSplit extends DoFn<String, Pair<String, Long>> { @Override @@ -99,9 +102,9 @@ public class CogroupIT { } public void run(Pipeline pipeline, PTypeFamily typeFamily) throws IOException { - String shakesInputPath = FileHelper.createTempCopyOf("shakes.txt"); - String maughamInputPath = FileHelper.createTempCopyOf("maugham.txt"); - File output = FileHelper.createOutputPath(); + String shakesInputPath = tmpDir.copyResourceFileName("shakes.txt"); + String maughamInputPath = tmpDir.copyResourceFileName("maugham.txt"); + File output = tmpDir.getFile("output"); PCollection<String> shakespeare = pipeline.read(From.textFile(shakesInputPath)); PCollection<String> maugham = pipeline.read(From.textFile(maughamInputPath)); @@ -118,7 +121,5 @@ public class CogroupIT { } } assertTrue(passed); - - output.deleteOnExit(); } } http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/6f549ed3/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java b/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java new file mode 100644 index 0000000..9218213 --- /dev/null +++ b/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java @@ -0,0 +1,105 @@ +package org.apache.crunch.test; + +import java.io.File; +import java.io.IOException; + +import org.apache.hadoop.fs.Path; +import org.junit.rules.ExternalResource; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; + +import com.google.common.io.Files; +import com.google.common.io.Resources; + + +/** + * Creates a temporary directory for a test case and destroys it + * afterwards. This provides a temporary directory like JUnit's + * {@link TemporaryFolder} but geared towards Hadoop applications. + * Unlike {@link TemporaryFolder}, it doesn't create any files or + * directories except for the root directory itself. + */ +public final class TemporaryPath extends ExternalResource { + private TemporaryFolder tmp = new TemporaryFolder(); + + @Override + public Statement apply(Statement base, Description description) { + return tmp.apply(base, description); + } + + /** + * Get the root directory which will be deleted automatically. + */ + public File getRootFile() { + return tmp.getRoot(); + } + + /** + * Get the root directory as a {@link Path}. + */ + public Path getRootPath() { + return toPath(tmp.getRoot()); + } + + /** + * Get the root directory as an absolute file name. + */ + public String getRootFileName() { + return tmp.getRoot().getAbsolutePath(); + } + + /** + * Get a {@link File} below the temporary directory. + */ + public File getFile(String fileName) { + return new File(getRootFile(), fileName); + } + + /** + * Get a {@link Path} below the temporary directory. + */ + public Path getPath(String fileName) { + return toPath(getFile(fileName)); + } + + /** + * Get an absolute file name below the temporary directory. + */ + public String getFileName(String fileName) { + return getFile(fileName).getAbsolutePath(); + } + + /** + * Copy a classpath resource to {@link File}. + */ + public File copyResourceFile(String resourceName) throws IOException { + File dest = new File(tmp.getRoot(), resourceName); + copy(resourceName, dest); + return dest; + } + + /** + * Copy a classpath resource to a {@link Path} + */ + public Path copyResourcePath(String resourceName) throws IOException { + return toPath(copyResourceFile(resourceName)); + } + + /** + * Copy a classpath resource returning its absolute file name. + */ + public String copyResourceFileName(String resourceName) throws IOException { + return copyResourceFile(resourceName).getAbsolutePath(); + } + + private void copy(String resourceName, File dest) throws IOException { + Files.copy( + Resources.newInputStreamSupplier(Resources.getResource(resourceName)), + dest); + } + + private static Path toPath(File file) { + return new Path(file.getAbsolutePath()); + } +} http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/6f549ed3/crunch/src/it/java/org/apache/crunch/test/TemporaryPathIT.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/test/TemporaryPathIT.java b/crunch/src/it/java/org/apache/crunch/test/TemporaryPathIT.java new file mode 100644 index 0000000..65018da --- /dev/null +++ b/crunch/src/it/java/org/apache/crunch/test/TemporaryPathIT.java @@ -0,0 +1,59 @@ +package org.apache.crunch.test; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +import java.io.File; +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocalFileSystem; +import org.junit.Rule; +import org.junit.Test; + + +public class TemporaryPathIT { + @Rule + public TemporaryPath tmpDir = new TemporaryPath(); + + @Test + public void testRoot() throws IOException { + assertThat(tmpDir.getRootFile().exists(), is(true)); + assertThat(new File(tmpDir.getRootFileName()).exists(), is(true)); + assertThat(getFs().exists(tmpDir.getRootPath()), is(true)); + } + + @Test + public void testFile() throws IOException { + assertThat(tmpDir.getFile("foo").getParentFile(), is(tmpDir.getRootFile())); + assertThat(tmpDir.getFile("foo").getName(), is("foo")); + assertThat(tmpDir.getFile("foo").exists(), is(false)); + } + + @Test + public void testPath() throws IOException { + assertThat(tmpDir.getPath("foo").getParent(), is(tmpDir.getRootPath())); + assertThat(tmpDir.getPath("foo").getName(), is("foo")); + assertThat(getFs().exists(tmpDir.getPath("foo")), is(false)); + } + + @Test + public void testFileName() { + assertThat(new File(tmpDir.getRootFileName()), is(tmpDir.getRootFile())); + assertThat(new File(tmpDir.getFileName("foo").toString()), is(tmpDir.getFile("foo"))); + } + + @Test + public void testCopyResource() throws IOException { + File dest = tmpDir.getFile("shakes.txt"); + assertThat(dest.exists(), is(false)); + + tmpDir.copyResourceFile("shakes.txt"); + assertThat(dest.exists(), is(true)); + } + + private LocalFileSystem getFs() throws IOException { + return FileSystem.getLocal(new Configuration()); + } +}
