MAPREDUCE-7077. Pipe mapreduce job fails with Permission denied for jobTokenPassword. (Akira Ajisaka via wangda)
Change-Id: Ie8f01425d58409fa3661f768205b7616128c8aa4 (cherry picked from commit 035e0f97ea44b0495707949a781d8792dcf6ea6b) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/995cba65 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/995cba65 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/995cba65 Branch: refs/heads/HDFS-7240 Commit: 995cba65fe29966583e36f9491d9a27b323918ae Parents: 9031a76 Author: Wangda Tan <wan...@apache.org> Authored: Thu Apr 12 14:33:33 2018 -0700 Committer: Wangda Tan <wan...@apache.org> Committed: Fri Apr 13 13:52:10 2018 -0700 ---------------------------------------------------------------------- .../apache/hadoop/mapred/pipes/Application.java | 5 ++--- .../hadoop/mapred/pipes/TestPipeApplication.java | 19 ++++++++----------- 2 files changed, 10 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/995cba65/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java index 5c8aab9..83d2509 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java @@ -45,7 +45,6 @@ import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapred.TaskAttemptID; import org.apache.hadoop.mapred.TaskLog; -import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.filecache.DistributedCache; import org.apache.hadoop.mapreduce.security.SecureShuffleUtils; @@ -104,8 +103,8 @@ class Application<K1 extends WritableComparable, V1 extends Writable, // This password is used as shared secret key between this application and // child pipes process byte[] password = jobToken.getPassword(); - String localPasswordFile = new File(conf.get(MRConfig.LOCAL_DIR)) - + Path.SEPARATOR + "jobTokenPassword"; + String localPasswordFile = new File(".") + Path.SEPARATOR + + "jobTokenPassword"; writePasswordToLocalFile(localPasswordFile, password, conf); env.put("hadoop.pipes.shared.secret.location", localPasswordFile); http://git-wip-us.apache.org/repos/asf/hadoop/blob/995cba65/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java index 13597e0..88d8f95 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java @@ -47,7 +47,6 @@ import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapred.IFile.Writer; -import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.security.TokenCache; import org.apache.hadoop.mapred.Counters; @@ -84,10 +83,10 @@ public class TestPipeApplication { public void testRunner() throws Exception { // clean old password files - JobConf conf = new JobConf(); - File[] psw = cleanTokenPasswordFile(conf); + File[] psw = cleanTokenPasswordFile(); try { RecordReader<FloatWritable, NullWritable> rReader = new ReaderPipesMapRunner(); + JobConf conf = new JobConf(); conf.set(Submitter.IS_JAVA_RR, "true"); // for stdour and stderror @@ -163,7 +162,7 @@ public class TestPipeApplication { TestTaskReporter reporter = new TestTaskReporter(); - File[] psw = cleanTokenPasswordFile(conf); + File[] psw = cleanTokenPasswordFile(); try { conf.set(MRJobConfig.TASK_ATTEMPT_ID, taskName); @@ -248,7 +247,7 @@ public class TestPipeApplication { JobConf conf = new JobConf(); - File[] psw = cleanTokenPasswordFile(conf); + File[] psw = cleanTokenPasswordFile(); System.setProperty("test.build.data", "target/tmp/build/TEST_SUBMITTER_MAPPER/data"); @@ -389,8 +388,8 @@ public class TestPipeApplication { @Test public void testPipesReduser() throws Exception { + File[] psw = cleanTokenPasswordFile(); JobConf conf = new JobConf(); - File[] psw = cleanTokenPasswordFile(conf); try { Token<AMRMTokenIdentifier> token = new Token<AMRMTokenIdentifier>( "user".getBytes(), "password".getBytes(), new Text("kind"), new Text( @@ -507,16 +506,14 @@ public class TestPipeApplication { } - private File[] cleanTokenPasswordFile(JobConf conf) throws Exception { + private File[] cleanTokenPasswordFile() throws Exception { File[] result = new File[2]; - result[0] = new File(conf.get(MRConfig.LOCAL_DIR) + Path.SEPARATOR - + "jobTokenPassword"); + result[0] = new File("./jobTokenPassword"); if (result[0].exists()) { FileUtil.chmod(result[0].getAbsolutePath(), "700"); assertTrue(result[0].delete()); } - result[1] = new File(conf.get(MRConfig.LOCAL_DIR) + Path.SEPARATOR - + ".jobTokenPassword.crc"); + result[1] = new File("./.jobTokenPassword.crc"); if (result[1].exists()) { FileUtil.chmod(result[1].getAbsolutePath(), "700"); result[1].delete(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
