Repository: hadoop Updated Branches: refs/heads/trunk 948b75807 -> 0ef8bbfd8
MAPREDUCE-6662. Clear ASF Warnings on test data files. Contributed by Vinayakumar B Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0ef8bbfd Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0ef8bbfd Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0ef8bbfd Branch: refs/heads/trunk Commit: 0ef8bbfd8791899c7cfed3dd9c1670182fd87575 Parents: 948b758 Author: Rohith Sharma K S <rohithsharm...@apache.org> Authored: Tue Mar 29 10:32:21 2016 +0800 Committer: Rohith Sharma K S <rohithsharm...@apache.org> Committed: Tue Mar 29 10:32:21 2016 +0800 ---------------------------------------------------------------------- .../mapreduce/v2/hs/TestHistoryFileManager.java | 5 +++-- .../apache/hadoop/mapred/TestFieldSelection.java | 5 +++-- .../mapred/TestMRTimelineEventHandling.java | 19 ++++++++++++------- .../hadoop/mapred/TestMiniMRChildTask.java | 16 ++++++++++------ .../hadoop/mapred/TestOldCombinerGrouping.java | 5 +++-- .../mapred/lib/aggregate/TestAggregates.java | 7 ++++--- .../mapreduce/TestNewCombinerGrouping.java | 5 +++-- .../lib/aggregate/TestMapReduceAggregates.java | 5 +++-- .../lib/input/TestCombineFileInputFormat.java | 4 +++- 9 files changed, 44 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ef8bbfd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java index 18e3e6a..aa2e979 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java @@ -71,8 +71,9 @@ public class TestHistoryFileManager { Configuration conf = new HdfsConfiguration(); Configuration conf2 = new HdfsConfiguration(); dfsCluster = new MiniDFSCluster.Builder(conf).build(); - conf2.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, - conf.get(MiniDFSCluster.HDFS_MINIDFS_BASEDIR) + "_2"); + conf2.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, conf.get( + MiniDFSCluster.HDFS_MINIDFS_BASEDIR, MiniDFSCluster.getBaseDirectory()) + + "_2"); dfsCluster2 = new MiniDFSCluster.Builder(conf2).build(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ef8bbfd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java index 239c239..29333b7b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java @@ -44,8 +44,9 @@ private static NumberFormat idFormat = NumberFormat.getInstance(); FileSystem fs = FileSystem.get(conf); int numOfInputLines = 10; - Path OUTPUT_DIR = new Path("build/test/output_for_field_selection_test"); - Path INPUT_DIR = new Path("build/test/input_for_field_selection_test"); + String baseDir = System.getProperty("test.build.data", "build/test/data"); + Path OUTPUT_DIR = new Path(baseDir + "/output_for_field_selection_test"); + Path INPUT_DIR = new Path(baseDir + "/input_for_field_selection_test"); String inputFile = "input.txt"; fs.delete(INPUT_DIR, true); fs.mkdirs(INPUT_DIR); http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ef8bbfd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java index a82007f..40ed9ad 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java @@ -97,9 +97,10 @@ public class TestMRTimelineEventHandling { + cluster.getApplicationHistoryServer().getPort()); TimelineStore ts = cluster.getApplicationHistoryServer() .getTimelineStore(); - - Path inDir = new Path("input"); - Path outDir = new Path("output"); + String localPathRoot = System.getProperty("test.build.data", + "build/test/data"); + Path inDir = new Path(localPathRoot, "input"); + Path outDir = new Path(localPathRoot, "output"); RunningJob job = UtilsForTests.runJobSucceed(new JobConf(conf), inDir, outDir); Assert.assertEquals(JobStatus.SUCCEEDED, @@ -155,8 +156,10 @@ public class TestMRTimelineEventHandling { TimelineStore ts = cluster.getApplicationHistoryServer() .getTimelineStore(); - Path inDir = new Path("input"); - Path outDir = new Path("output"); + String localPathRoot = System.getProperty("test.build.data", + "build/test/data"); + Path inDir = new Path(localPathRoot, "input"); + Path outDir = new Path(localPathRoot, "output"); RunningJob job = UtilsForTests.runJobSucceed(new JobConf(conf), inDir, outDir); Assert.assertEquals(JobStatus.SUCCEEDED, @@ -195,8 +198,10 @@ public class TestMRTimelineEventHandling { TimelineStore ts = cluster.getApplicationHistoryServer() .getTimelineStore(); - Path inDir = new Path("input"); - Path outDir = new Path("output"); + String localPathRoot = System.getProperty("test.build.data", + "build/test/data"); + Path inDir = new Path(localPathRoot, "input"); + Path outDir = new Path(localPathRoot, "output"); conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, false); RunningJob job = http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ef8bbfd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java index 74be8a7..cbeeccf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java @@ -436,8 +436,10 @@ public class TestMiniMRChildTask { JobConf conf = new JobConf(mr.getConfig()); // initialize input, output directories - Path inDir = new Path("input"); - Path outDir = new Path("output"); + Path rootDir = new Path(System.getProperty("test.build.data", + "build/test/data")); + Path inDir = new Path(rootDir, "input"); + Path outDir = new Path(rootDir, "output"); String input = "The input"; // set config to use the ExecutionEnvCheckMapClass map class @@ -462,9 +464,10 @@ public class TestMiniMRChildTask { public void testTaskEnv(){ try { JobConf conf = new JobConf(mr.getConfig()); + String baseDir = System.getProperty("test.build.data", "build/test/data"); // initialize input, output directories - Path inDir = new Path("testing/wc/input1"); - Path outDir = new Path("testing/wc/output1"); + Path inDir = new Path(baseDir + "/testing/wc/input1"); + Path outDir = new Path(baseDir + "/testing/wc/output1"); FileSystem outFs = outDir.getFileSystem(conf); runTestTaskEnv(conf, inDir, outDir, false); outFs.delete(outDir, true); @@ -485,9 +488,10 @@ public class TestMiniMRChildTask { public void testTaskOldEnv(){ try { JobConf conf = new JobConf(mr.getConfig()); + String baseDir = System.getProperty("test.build.data", "build/test/data"); // initialize input, output directories - Path inDir = new Path("testing/wc/input1"); - Path outDir = new Path("testing/wc/output1"); + Path inDir = new Path(baseDir + "/testing/wc/input1"); + Path outDir = new Path(baseDir + "/testing/wc/output1"); FileSystem outFs = outDir.getFileSystem(conf); runTestTaskEnv(conf, inDir, outDir, true); outFs.delete(outDir, true); http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ef8bbfd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java index 80a6702..046c2d3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java @@ -37,8 +37,9 @@ import java.util.Set; import java.util.UUID; public class TestOldCombinerGrouping { - private static String TEST_ROOT_DIR = - new File("build", UUID.randomUUID().toString()).getAbsolutePath(); + private static String TEST_ROOT_DIR = new File(System.getProperty( + "test.build.data", "build/test/data"), UUID.randomUUID().toString()) + .getAbsolutePath(); public static class Map implements Mapper<LongWritable, Text, Text, LongWritable> { http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ef8bbfd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java index 6da96ce..e2fdd42 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java @@ -46,8 +46,9 @@ public class TestAggregates extends TestCase { FileSystem fs = FileSystem.get(conf); int numOfInputLines = 20; - Path OUTPUT_DIR = new Path("build/test/output_for_aggregates_test"); - Path INPUT_DIR = new Path("build/test/input_for_aggregates_test"); + String baseDir = System.getProperty("test.build.data", "build/test/data"); + Path OUTPUT_DIR = new Path(baseDir + "/output_for_aggregates_test"); + Path INPUT_DIR = new Path(baseDir + "/input_for_aggregates_test"); String inputFile = "input.txt"; fs.delete(INPUT_DIR, true); fs.mkdirs(INPUT_DIR); @@ -115,7 +116,7 @@ public class TestAggregates extends TestCase { outdata = outdata.substring(0, expectedOutput.toString().length()); assertEquals(expectedOutput.toString(),outdata); - //fs.delete(OUTPUT_DIR); + fs.delete(OUTPUT_DIR, true); fs.delete(INPUT_DIR, true); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ef8bbfd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java index ab9ddbb..c2054f1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java @@ -39,8 +39,9 @@ import java.util.Set; import java.util.UUID; public class TestNewCombinerGrouping { - private static String TEST_ROOT_DIR = - new File("build", UUID.randomUUID().toString()).getAbsolutePath(); + private static String TEST_ROOT_DIR = new File(System.getProperty( + "test.build.data", "build/test/data"), UUID.randomUUID().toString()) + .getAbsolutePath(); public static class Map extends Mapper<LongWritable, Text, Text, LongWritable> { http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ef8bbfd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java index f24dffe..788ad41 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java @@ -51,8 +51,9 @@ public class TestMapReduceAggregates extends TestCase { FileSystem fs = FileSystem.get(conf); int numOfInputLines = 20; - Path OUTPUT_DIR = new Path("build/test/output_for_aggregates_test"); - Path INPUT_DIR = new Path("build/test/input_for_aggregates_test"); + String baseDir = System.getProperty("test.build.data", "build/test/data"); + Path OUTPUT_DIR = new Path(baseDir + "/output_for_aggregates_test"); + Path INPUT_DIR = new Path(baseDir + "/input_for_aggregates_test"); String inputFile = "input.txt"; fs.delete(INPUT_DIR, true); fs.mkdirs(INPUT_DIR); http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ef8bbfd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java index b49f2d8..1fca5c9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java @@ -1772,7 +1772,9 @@ public class TestCombineFileInputFormat { // default fs path assertEquals(DUMMY_FS_URI, FileSystem.getDefaultUri(conf).toString()); // add a local file - Path localPath = new Path("testFile1"); + String localPathRoot = System.getProperty("test.build.data", + "build/test/data"); + Path localPath = new Path(localPathRoot, "testFile1"); FileSystem lfs = FileSystem.getLocal(conf); FSDataOutputStream dos = lfs.create(localPath); dos.writeChars("Local file for CFIF");
