TEZ-3131. Support a way to override test_root_dir for FaultToleranceTestRunner. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/44ca2295 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/44ca2295 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/44ca2295 Branch: refs/heads/TEZ-2980 Commit: 44ca2295f8209dd83041e426d0dca95721461de6 Parents: f38e23c Author: Hitesh Shah <hit...@apache.org> Authored: Mon Feb 22 17:13:16 2016 -0800 Committer: Hitesh Shah <hit...@apache.org> Committed: Mon Feb 22 17:13:16 2016 -0800 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../library/vertexmanager/ShuffleVertexManager.java | 4 +++- .../apache/tez/test/FaultToleranceTestRunner.java | 15 ++++++++++++--- 3 files changed, 16 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/44ca2295/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index dc27f93..2311c55 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES TEZ-3029. Add an onError method to service plugin contexts. ALL CHANGES: + TEZ-3131. Support a way to override test_root_dir for FaultToleranceTestRunner. TEZ-3067. Links to tez configs documentation should be bubbled up to top-level release page. TEZ-3123. Containers can get re-used even with conflicting local resources. TEZ-3117. Deadlock in Edge and Vertex code http://git-wip-us.apache.org/repos/asf/tez/blob/44ca2295/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java index 410ad73..ea00532 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java @@ -1004,7 +1004,9 @@ public class ShuffleVertexManager extends VertexManagerPlugin { throw new IllegalArgumentException( "Invalid values for slowStartMinSrcCompletionFraction" + "/slowStartMaxSrcCompletionFraction. Min cannot be < 0, max cannot be > 1," - + " and max cannot be < min."); + + " and max cannot be < min." + + ", configuredMin=" + slowStartMinSrcCompletionFraction + + ", configuredMax=" + slowStartMaxSrcCompletionFraction); } enableAutoParallelism = conf http://git-wip-us.apache.org/repos/asf/tez/blob/44ca2295/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java ---------------------------------------------------------------------- diff --git a/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java b/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java index 84e5b97..94242a7 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java +++ b/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java @@ -44,7 +44,8 @@ import org.apache.hadoop.util.GenericOptionsParser; */ public class FaultToleranceTestRunner { - static String TEST_ROOT_DIR = "tmp"; + static String DEFAULT_FT_STAGING_DIR = "tmp"; + static String FT_STAGING_DIR = "tez.test-fault-tolerance.staging-dir"; Configuration conf = null; TezClient tezSession = null; Resource defaultResource = Resource.newInstance(100, 0); @@ -57,8 +58,16 @@ public class FaultToleranceTestRunner { tezConf = new TezConfiguration(new YarnConfiguration(this.conf)); } FileSystem defaultFs = FileSystem.get(tezConf); - - Path remoteStagingDir = defaultFs.makeQualified(new Path(TEST_ROOT_DIR, String + + Path testRootDir; + if (tezConf.get(FT_STAGING_DIR) != null + && !tezConf.get(FT_STAGING_DIR).isEmpty()) { + testRootDir = new Path(tezConf.get(FT_STAGING_DIR)); + } else { + testRootDir = new Path(DEFAULT_FT_STAGING_DIR); + } + + Path remoteStagingDir = defaultFs.makeQualified(new Path(testRootDir, String .valueOf(new Random().nextInt(100000)))); TezClientUtils.ensureStagingDirExists(tezConf, remoteStagingDir);
