Author: szetszwo
Date: Tue Nov 27 23:02:00 2012
New Revision: 1414456
URL: http://svn.apache.org/viewvc?rev=1414456&view=rev
Log:
Merge r1412283 through r1414454 from trunk.
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt
(contents, props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/conf/ (props
changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsAboutPage.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/HistoryInfo.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestBinaryTokenFile.java
Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1412283-1414454
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt?rev=1414456&r1=1414455&r2=1414456&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt Tue
Nov 27 23:02:00 2012
@@ -586,6 +586,9 @@ Release 0.23.6 - UNRELEASED
IMPROVEMENTS
+ MAPREDUCE-4811. JobHistoryServer should show when it was started in WebUI
+ About page (Ravi Prakash via jlowe)
+
OPTIMIZATIONS
BUG FIXES
@@ -593,6 +596,8 @@ Release 0.23.6 - UNRELEASED
MAPREDUCE-4802. Takes a long time to load the task list on the AM for
large jobs (Ravi Prakash via bobby)
+ MAPREDUCE-4764. repair TestBinaryTokenFile (Ivan A. Veselovsky via bobby)
+
Release 0.23.5 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1412283-1414454
Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1412283-1414454
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1412283-1414454
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java?rev=1414456&r1=1414455&r2=1414456&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
Tue Nov 27 23:02:00 2012
@@ -51,6 +51,8 @@ public class JobHistoryServer extends Co
*/
public static final int SHUTDOWN_HOOK_PRIORITY = 30;
+ public static final long historyServerTimeStamp = System.currentTimeMillis();
+
private static final Log LOG = LogFactory.getLog(JobHistoryServer.class);
private HistoryContext historyContext;
private HistoryClientService clientService;
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsAboutPage.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsAboutPage.java?rev=1414456&r1=1414455&r2=1414456&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsAboutPage.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsAboutPage.java
Tue Nov 27 23:02:00 2012
@@ -21,7 +21,9 @@ package org.apache.hadoop.mapreduce.v2.h
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
+import org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer;
import org.apache.hadoop.mapreduce.v2.hs.webapp.dao.HistoryInfo;
+import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
@@ -47,7 +49,9 @@ public class HsAboutPage extends HsView
@Override protected Class<? extends SubView> content() {
HistoryInfo info = new HistoryInfo();
info("History Server").
- _("BuildVersion", info.getHadoopBuildVersion() + " on " +
info.getHadoopVersionBuiltOn());
+ _("BuildVersion", info.getHadoopBuildVersion()
+ + " on " + info.getHadoopVersionBuiltOn()).
+ _("History Server started on", Times.format(info.getStartedOn()));
return InfoBlock.class;
}
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/HistoryInfo.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/HistoryInfo.java?rev=1414456&r1=1414455&r2=1414456&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/HistoryInfo.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/HistoryInfo.java
Tue Nov 27 23:02:00 2012
@@ -22,17 +22,20 @@ import javax.xml.bind.annotation.XmlAcce
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
+import org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer;
import org.apache.hadoop.util.VersionInfo;
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
public class HistoryInfo {
+ protected long startedOn;
protected String hadoopVersion;
protected String hadoopBuildVersion;
protected String hadoopVersionBuiltOn;
public HistoryInfo() {
+ this.startedOn = JobHistoryServer.historyServerTimeStamp;
this.hadoopVersion = VersionInfo.getVersion();
this.hadoopBuildVersion = VersionInfo.getBuildVersion();
this.hadoopVersionBuiltOn = VersionInfo.getDate();
@@ -50,4 +53,8 @@ public class HistoryInfo {
return this.hadoopVersionBuiltOn;
}
+ public long getStartedOn() {
+ return this.startedOn;
+ }
+
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java?rev=1414456&r1=1414455&r2=1414456&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java
Tue Nov 27 23:02:00 2012
@@ -36,6 +36,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.hs.HistoryContext;
import org.apache.hadoop.mapreduce.v2.hs.JobHistory;
+import org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer;
import org.apache.hadoop.mapreduce.v2.hs.webapp.dao.JobsInfo;
import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.Clock;
@@ -344,21 +345,24 @@ public class TestHsWebServices extends J
}
public void verifyHsInfoGeneric(String hadoopVersionBuiltOn,
- String hadoopBuildVersion, String hadoopVersion) {
+ String hadoopBuildVersion, String hadoopVersion, long startedon) {
WebServicesTestUtils.checkStringMatch("hadoopVersionBuiltOn",
VersionInfo.getDate(), hadoopVersionBuiltOn);
WebServicesTestUtils.checkStringMatch("hadoopBuildVersion",
VersionInfo.getBuildVersion(), hadoopBuildVersion);
WebServicesTestUtils.checkStringMatch("hadoopVersion",
VersionInfo.getVersion(), hadoopVersion);
+ assertEquals("startedOn doesn't match: ",
+ JobHistoryServer.historyServerTimeStamp, startedon);
}
public void verifyHSInfo(JSONObject info, TestAppContext ctx)
throws JSONException {
- assertEquals("incorrect number of elements", 3, info.length());
+ assertEquals("incorrect number of elements", 4, info.length());
verifyHsInfoGeneric(info.getString("hadoopVersionBuiltOn"),
- info.getString("hadoopBuildVersion"), info.getString("hadoopVersion"));
+ info.getString("hadoopBuildVersion"), info.getString("hadoopVersion"),
+ info.getLong("startedOn"));
}
public void verifyHSInfoXML(String xml, TestAppContext ctx)
@@ -376,7 +380,8 @@ public class TestHsWebServices extends J
verifyHsInfoGeneric(
WebServicesTestUtils.getXmlString(element, "hadoopVersionBuiltOn"),
WebServicesTestUtils.getXmlString(element, "hadoopBuildVersion"),
- WebServicesTestUtils.getXmlString(element, "hadoopVersion"));
+ WebServicesTestUtils.getXmlString(element, "hadoopVersion"),
+ WebServicesTestUtils.getXmlLong(element, "startedOn"));
}
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestBinaryTokenFile.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestBinaryTokenFile.java?rev=1414456&r1=1414455&r2=1414456&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestBinaryTokenFile.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestBinaryTokenFile.java
Tue Nov 27 23:02:00 2012
@@ -35,26 +35,28 @@ import org.apache.hadoop.hdfs.MiniDFSClu
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.SleepJob;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
+import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster;
import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
-@SuppressWarnings("deprecation")
-@Ignore
public class TestBinaryTokenFile {
+ private static final String KEY_SECURITY_TOKEN_FILE_NAME =
"key-security-token-file";
+ private static final String DELEGATION_TOKEN_KEY = "Hdfs";
+
// my sleep class
static class MySleepMapper extends SleepJob.SleepMapper {
/**
@@ -63,29 +65,65 @@ public class TestBinaryTokenFile {
@Override
public void map(IntWritable key, IntWritable value, Context context)
throws IOException, InterruptedException {
- // get token storage and a key
- Credentials ts = context.getCredentials();
- Collection<Token<? extends TokenIdentifier>> dts = ts.getAllTokens();
+ // get context token storage:
+ final Credentials contextCredentials = context.getCredentials();
+ final Collection<Token<? extends TokenIdentifier>>
contextTokenCollection = contextCredentials.getAllTokens();
+ for (Token<? extends TokenIdentifier> t : contextTokenCollection) {
+ System.out.println("Context token: [" + t + "]");
+ }
+ if (contextTokenCollection.size() != 2) { // one job token and one
delegation token
+ // fail the test:
+ throw new RuntimeException("Exactly 2 tokens are expected in the
contextTokenCollection: " +
+ "one job token and one delegation token, but was found
" + contextTokenCollection.size() + " tokens.");
+ }
- if(dts.size() != 2) { // one job token and one delegation token
- throw new RuntimeException("tokens are not available"); // fail the
test
+ final Token<? extends TokenIdentifier> dt =
contextCredentials.getToken(new Text(DELEGATION_TOKEN_KEY));
+ if (dt == null) {
+ throw new RuntimeException("Token for key ["+DELEGATION_TOKEN_KEY+"]
not found in the job context.");
}
- Token<? extends TokenIdentifier> dt = ts.getToken(new Text("Hdfs"));
+ String tokenFile0 =
context.getConfiguration().get(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY);
+ if (tokenFile0 != null) {
+ throw new RuntimeException("Token file key
["+MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY+"] found in the configuration.
It should have been removed from the configuration.");
+ }
- //Verify that dt is same as the token in the file
- String tokenFile = context.getConfiguration().get(
- "mapreduce.job.credentials.binary");
- Credentials cred = new Credentials();
- cred.readTokenStorageStream(new DataInputStream(new FileInputStream(
+ final String tokenFile =
context.getConfiguration().get(KEY_SECURITY_TOKEN_FILE_NAME);
+ if (tokenFile == null) {
+ throw new RuntimeException("Token file key
["+KEY_SECURITY_TOKEN_FILE_NAME+"] not found in the job configuration.");
+ }
+ final Credentials binaryCredentials = new Credentials();
+ binaryCredentials.readTokenStorageStream(new DataInputStream(new
FileInputStream(
tokenFile)));
- for (Token<? extends TokenIdentifier> t : cred.getAllTokens()) {
- if (!dt.equals(t)) {
- throw new RuntimeException(
- "Delegation token in job is not same as the token passed in
file."
- + " tokenInFile=" + t + ", dt=" + dt);
- }
+ final Collection<Token<? extends TokenIdentifier>> binaryTokenCollection
= binaryCredentials.getAllTokens();
+ if (binaryTokenCollection.size() != 1) {
+ throw new RuntimeException("The token collection read from file
["+tokenFile+"] must have size = 1.");
+ }
+ final Token<? extends TokenIdentifier> binTok =
binaryTokenCollection.iterator().next();
+ System.out.println("The token read from binary file: t = [" + binTok +
"]");
+ // Verify that dt is same as the token in the file:
+ if (!dt.equals(binTok)) {
+ throw new RuntimeException(
+ "Delegation token in job is not same as the token passed in
file:"
+ + " tokenInFile=[" + binTok + "], dt=[" + dt + "].");
+ }
+
+ // Now test the user tokens.
+ final UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ // Print all the UGI tokens for diagnostic purposes:
+ final Collection<Token<? extends TokenIdentifier>> ugiTokenCollection =
ugi.getTokens();
+ for (Token<? extends TokenIdentifier> t: ugiTokenCollection) {
+ System.out.println("UGI token: [" + t + "]");
+ }
+ final Token<? extends TokenIdentifier> ugiToken
+ = ugi.getCredentials().getToken(new Text(DELEGATION_TOKEN_KEY));
+ if (ugiToken == null) {
+ throw new RuntimeException("Token for key ["+DELEGATION_TOKEN_KEY+"]
not found among the UGI tokens.");
+ }
+ if (!ugiToken.equals(binTok)) {
+ throw new RuntimeException(
+ "UGI token is not same as the token passed in binary file:"
+ + " tokenInBinFile=[" + binTok + "], ugiTok=[" + ugiToken +
"].");
}
super.map(key, value, context);
@@ -118,13 +156,20 @@ public class TestBinaryTokenFile {
TokenCache.obtainTokensForNamenodesInternal(cred1, new Path[] { p1 },
job.getConfiguration());
for (Token<? extends TokenIdentifier> t : cred1.getAllTokens()) {
- cred2.addToken(new Text("Hdfs"), t);
+ cred2.addToken(new Text(DELEGATION_TOKEN_KEY), t);
}
DataOutputStream os = new DataOutputStream(new FileOutputStream(
binaryTokenFileName.toString()));
- cred2.writeTokenStorageToStream(os);
- os.close();
- job.getConfiguration().set("mapreduce.job.credentials.binary",
+ try {
+ cred2.writeTokenStorageToStream(os);
+ } finally {
+ os.close();
+ }
+
job.getConfiguration().set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY,
+ binaryTokenFileName.toString());
+ // NB: the MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY key now gets
deleted from config,
+ // so it's not accessible in the job's config. So, we use another key
to pass the file name into the job configuration:
+ job.getConfiguration().set(KEY_SECURITY_TOKEN_FILE_NAME,
binaryTokenFileName.toString());
} catch (IOException e) {
Assert.fail("Exception " + e);
@@ -132,39 +177,53 @@ public class TestBinaryTokenFile {
}
}
- private static MiniMRCluster mrCluster;
+ private static MiniMRYarnCluster mrCluster;
private static MiniDFSCluster dfsCluster;
+
private static final Path TEST_DIR =
new Path(System.getProperty("test.build.data","/tmp"));
private static final Path binaryTokenFileName = new Path(TEST_DIR,
"tokenFile.binary");
- private static int numSlaves = 1;
- private static JobConf jConf;
+
+ private static final int numSlaves = 1; // num of data nodes
+ private static final int noOfNMs = 1;
+
private static Path p1;
@BeforeClass
public static void setUp() throws Exception {
- Configuration conf = new Configuration();
- dfsCluster = new MiniDFSCluster(conf, numSlaves, true, null);
- jConf = new JobConf(conf);
- mrCluster = new MiniMRCluster(0, 0, numSlaves,
- dfsCluster.getFileSystem().getUri().toString(), 1, null, null, null,
- jConf);
+ final Configuration conf = new Configuration();
+
+ conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
+ conf.set(YarnConfiguration.RM_PRINCIPAL, "jt_id/" +
SecurityUtil.HOSTNAME_PATTERN + "@APACHE.ORG");
+
+ final MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
+ builder.checkExitOnShutdown(true);
+ builder.numDataNodes(numSlaves);
+ builder.format(true);
+ builder.racks(null);
+ dfsCluster = builder.build();
+
+ mrCluster = new MiniMRYarnCluster(TestBinaryTokenFile.class.getName(),
noOfNMs);
+ mrCluster.init(conf);
+ mrCluster.start();
-
NameNodeAdapter.getDtSecretManager(dfsCluster.getNamesystem()).startThreads();
- FileSystem fs = dfsCluster.getFileSystem();
+
NameNodeAdapter.getDtSecretManager(dfsCluster.getNamesystem()).startThreads();
+ FileSystem fs = dfsCluster.getFileSystem();
p1 = new Path("file1");
p1 = fs.makeQualified(p1);
}
@AfterClass
public static void tearDown() throws Exception {
- if(mrCluster != null)
- mrCluster.shutdown();
- mrCluster = null;
- if(dfsCluster != null)
+ if(mrCluster != null) {
+ mrCluster.stop();
+ mrCluster = null;
+ }
+ if(dfsCluster != null) {
dfsCluster.shutdown();
- dfsCluster = null;
+ dfsCluster = null;
+ }
}
/**
@@ -173,31 +232,24 @@ public class TestBinaryTokenFile {
*/
@Test
public void testBinaryTokenFile() throws IOException {
-
- System.out.println("running dist job");
-
- // make sure JT starts
- jConf = mrCluster.createJobConf();
+ Configuration conf = mrCluster.getConfig();
// provide namenodes names for the job to get the delegation tokens for
- String nnUri = dfsCluster.getURI(0).toString();
- jConf.set(MRJobConfig.JOB_NAMENODES, nnUri + "," + nnUri);
- // job tracker principla id..
- jConf.set(JTConfig.JT_USER_NAME, "jt_id");
+ final String nnUri = dfsCluster.getURI(0).toString();
+ conf.set(MRJobConfig.JOB_NAMENODES, nnUri + "," + nnUri);
// using argument to pass the file name
- String[] args = {
+ final String[] args = {
"-m", "1", "-r", "1", "-mt", "1", "-rt", "1"
};
-
int res = -1;
try {
- res = ToolRunner.run(jConf, new MySleepJob(), args);
+ res = ToolRunner.run(conf, new MySleepJob(), args);
} catch (Exception e) {
- System.out.println("Job failed with" + e.getLocalizedMessage());
+ System.out.println("Job failed with " + e.getLocalizedMessage());
e.printStackTrace(System.out);
fail("Job failed");
}
- assertEquals("dist job res is not 0", res, 0);
+ assertEquals("dist job res is not 0:", 0, res);
}
}
