CRUNCH-24: Clean up data after integration test. Redirect more test output to self-cleaning temporary directories.
Signed-off-by: jwills <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/e540459e Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/e540459e Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/e540459e Branch: refs/heads/master Commit: e540459ee2f4c3b7f29989c16cb9aa19278fd3fb Parents: be357f2 Author: Matthias Friedrich <[email protected]> Authored: Wed Jul 25 18:58:44 2012 +0200 Committer: jwills <[email protected]> Committed: Wed Jul 25 22:23:38 2012 -0700 ---------------------------------------------------------------------- .../it/java/org/apache/crunch/CollectionsIT.java | 2 - crunch/src/it/java/org/apache/crunch/MapsIT.java | 19 ++++++-------- .../java/org/apache/crunch/WordCountHBaseIT.java | 3 +- .../crunch/impl/mem/MemPipelineFileWritingIT.java | 8 ++++- .../apache/crunch/io/CompositePathIterableIT.java | 16 +++++++----- .../java/org/apache/crunch/test/TemporaryPath.java | 3 +- 6 files changed, 27 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/e540459e/crunch/src/it/java/org/apache/crunch/CollectionsIT.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/CollectionsIT.java b/crunch/src/it/java/org/apache/crunch/CollectionsIT.java index e6be525..c9ff2a9 100644 --- a/crunch/src/it/java/org/apache/crunch/CollectionsIT.java +++ b/crunch/src/it/java/org/apache/crunch/CollectionsIT.java @@ -24,7 +24,6 @@ import java.util.Collection; import org.apache.crunch.impl.mem.MemPipeline; import org.apache.crunch.impl.mr.MRPipeline; -import org.apache.crunch.impl.mr.run.RuntimeParameters; import org.apache.crunch.test.FileHelper; import org.apache.crunch.test.TemporaryPath; import org.apache.crunch.types.PTypeFamily; @@ -33,7 +32,6 @@ import org.apache.crunch.types.writable.WritableTypeFamily; import org.apache.hadoop.conf.Configuration; import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/e540459e/crunch/src/it/java/org/apache/crunch/MapsIT.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/MapsIT.java b/crunch/src/it/java/org/apache/crunch/MapsIT.java index e0911c9..3b9fb37 100644 --- a/crunch/src/it/java/org/apache/crunch/MapsIT.java +++ b/crunch/src/it/java/org/apache/crunch/MapsIT.java @@ -17,41 +17,38 @@ */ package org.apache.crunch; +import java.io.Serializable; import java.util.Map; import org.apache.crunch.impl.mr.MRPipeline; -import org.apache.crunch.impl.mr.run.RuntimeParameters; -import org.apache.crunch.test.FileHelper; +import org.apache.crunch.test.TemporaryPath; import org.apache.crunch.types.PTypeFamily; import org.apache.crunch.types.avro.AvroTypeFamily; import org.apache.crunch.types.writable.WritableTypeFamily; import org.apache.hadoop.conf.Configuration; import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; public class MapsIT { @Rule - public TemporaryFolder temporaryFolder = new TemporaryFolder(); + public TemporaryPath tmpDir = new TemporaryPath(); @Test public void testWritables() throws Exception { - run(WritableTypeFamily.getInstance(), temporaryFolder.newFolder().getAbsolutePath()); + run(WritableTypeFamily.getInstance(), tmpDir); } @Test public void testAvros() throws Exception { - run(AvroTypeFamily.getInstance(), temporaryFolder.newFolder().getAbsolutePath()); + run(AvroTypeFamily.getInstance(), tmpDir); } - public static void run(PTypeFamily typeFamily, String path) throws Exception { - Configuration config = new Configuration(); - config.set(RuntimeParameters.TMP_DIR, path); - Pipeline pipeline = new MRPipeline(MapsIT.class, config); - String shakesInputPath = FileHelper.createTempCopyOf("shakes.txt"); + public static void run(PTypeFamily typeFamily, TemporaryPath tmpDir) throws Exception { + Pipeline pipeline = new MRPipeline(MapsIT.class, tmpDir.setTempLoc(new Configuration())); + String shakesInputPath = tmpDir.copyResourceFileName("shakes.txt"); PCollection<String> shakespeare = pipeline.readTextFile(shakesInputPath); Iterable<Pair<String, Map<String, Long>>> output = shakespeare .parallelDo(new DoFn<String, Pair<String, Map<String, Long>>>() { http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/e540459e/crunch/src/it/java/org/apache/crunch/WordCountHBaseIT.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/WordCountHBaseIT.java b/crunch/src/it/java/org/apache/crunch/WordCountHBaseIT.java index 9704c3b..9d42ee0 100644 --- a/crunch/src/it/java/org/apache/crunch/WordCountHBaseIT.java +++ b/crunch/src/it/java/org/apache/crunch/WordCountHBaseIT.java @@ -92,7 +92,8 @@ public class WordCountHBaseIT { @Before public void setUp() throws Exception { Configuration conf = hbaseTestUtil.getConfiguration(); - conf.set("hadoop.log.dir", tmpDir.getRootFileName()); + conf.set("hadoop.log.dir", tmpDir.getFileName("logs")); + conf.set("hadoop.tmp.dir", tmpDir.getFileName("hadoop-tmp")); conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1"); conf.setInt("hbase.master.info.port", -1); conf.setInt("hbase.regionserver.info.port", -1); http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/e540459e/crunch/src/it/java/org/apache/crunch/impl/mem/MemPipelineFileWritingIT.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/impl/mem/MemPipelineFileWritingIT.java b/crunch/src/it/java/org/apache/crunch/impl/mem/MemPipelineFileWritingIT.java index f4a0a90..db0e8ce 100644 --- a/crunch/src/it/java/org/apache/crunch/impl/mem/MemPipelineFileWritingIT.java +++ b/crunch/src/it/java/org/apache/crunch/impl/mem/MemPipelineFileWritingIT.java @@ -25,6 +25,8 @@ import java.util.List; import org.apache.crunch.PCollection; import org.apache.crunch.Pipeline; +import org.apache.crunch.test.TemporaryPath; +import org.junit.Rule; import org.junit.Test; import com.google.common.base.Charsets; @@ -32,10 +34,12 @@ import com.google.common.collect.ImmutableList; import com.google.common.io.Files; public class MemPipelineFileWritingIT { + @Rule + public TemporaryPath baseTmpDir = new TemporaryPath(); + @Test public void testMemPipelineFileWriter() throws Exception { - File tmpDir = Files.createTempDir(); - tmpDir.delete(); + File tmpDir = baseTmpDir.getRootFile(); Pipeline p = MemPipeline.getInstance(); PCollection<String> lines = MemPipeline.collectionOf("hello", "world"); p.writeTextFile(lines, tmpDir.getAbsolutePath()); http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/e540459e/crunch/src/it/java/org/apache/crunch/io/CompositePathIterableIT.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/io/CompositePathIterableIT.java b/crunch/src/it/java/org/apache/crunch/io/CompositePathIterableIT.java index 0f1a45e..c55287c 100644 --- a/crunch/src/it/java/org/apache/crunch/io/CompositePathIterableIT.java +++ b/crunch/src/it/java/org/apache/crunch/io/CompositePathIterableIT.java @@ -26,17 +26,20 @@ import java.io.IOException; import org.apache.crunch.io.text.TextFileReaderFactory; import org.apache.crunch.test.FileHelper; +import org.apache.crunch.test.TemporaryPath; import org.apache.crunch.types.writable.Writables; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; +import org.junit.Rule; import org.junit.Test; import com.google.common.collect.Lists; -import com.google.common.io.Files; public class CompositePathIterableIT { + @Rule + public TemporaryPath tmpDir = new TemporaryPath(); @Test public void testCreate_FilePresent() throws IOException { @@ -53,12 +56,12 @@ public class CompositePathIterableIT { @Test public void testCreate_DirectoryPresentButNoFiles() throws IOException { - String inputFilePath = Files.createTempDir().getAbsolutePath(); + Path emptyInputDir = tmpDir.getRootPath(); Configuration conf = new Configuration(); LocalFileSystem local = FileSystem.getLocal(conf); - Iterable<String> iterable = CompositePathIterable.create(local, new Path(inputFilePath), + Iterable<String> iterable = CompositePathIterable.create(local, emptyInputDir, new TextFileReaderFactory<String>(Writables.strings(), conf)); assertTrue(Lists.newArrayList(iterable).isEmpty()); @@ -66,16 +69,15 @@ public class CompositePathIterableIT { @Test(expected = IOException.class) public void testCreate_DirectoryNotPresent() throws IOException { - File inputFileDir = Files.createTempDir(); - inputFileDir.delete(); + File nonExistentDir = tmpDir.getFile("not-there"); // Sanity check - assertFalse(inputFileDir.exists()); + assertFalse(nonExistentDir.exists()); Configuration conf = new Configuration(); LocalFileSystem local = FileSystem.getLocal(conf); - CompositePathIterable.create(local, new Path(inputFileDir.getAbsolutePath()), new TextFileReaderFactory<String>( + CompositePathIterable.create(local, new Path(nonExistentDir.getAbsolutePath()), new TextFileReaderFactory<String>( Writables.strings(), conf)); } http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/e540459e/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java b/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java index 7bdc91a..dd4f505 100644 --- a/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java +++ b/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java @@ -103,7 +103,8 @@ public final class TemporaryPath extends ExternalResource { } public Configuration setTempLoc(Configuration config) throws IOException { - config.set(RuntimeParameters.TMP_DIR, tmp.newFolder().getAbsolutePath()); + config.set(RuntimeParameters.TMP_DIR, getRootFileName()); + config.set("hadoop.tmp.dir", getFileName("hadoop-tmp")); return config; } }
