Author: ddas
Date: Fri Jul 23 00:05:32 2010
New Revision: 966911
URL: http://svn.apache.org/viewvc?rev=966911&view=rev
Log:
HADOOP-6861. Adds new non-static methods in Credentials to read and write token
storage file. Contributed by Jitendra Pandey & Owen O'Malley.
Modified:
hadoop/common/trunk/CHANGES.txt
hadoop/common/trunk/src/java/org/apache/hadoop/io/WritableUtils.java
hadoop/common/trunk/src/java/org/apache/hadoop/security/Credentials.java
hadoop/common/trunk/src/java/org/apache/hadoop/security/UserGroupInformation.java
hadoop/common/trunk/src/java/org/apache/hadoop/util/GenericOptionsParser.java
hadoop/common/trunk/src/test/core/org/apache/hadoop/util/TestGenericOptionsParser.java
Modified: hadoop/common/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=966911&r1=966910&r2=966911&view=diff
==============================================================================
--- hadoop/common/trunk/CHANGES.txt (original)
+++ hadoop/common/trunk/CHANGES.txt Fri Jul 23 00:05:32 2010
@@ -80,6 +80,9 @@ Trunk (unreleased changes)
same principal. Now the principal name is a pattern that has _HOST in it.
(Kan Zhang & Jitendra Pandey via ddas)
+ HADOOP-6861. Adds new non-static methods in Credentials to read and
+ write token storage file. (Jitendra Pandey & Owen O'Malley via ddas)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/io/WritableUtils.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/io/WritableUtils.java?rev=966911&r1=966910&r2=966911&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/io/WritableUtils.java
(original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/io/WritableUtils.java Fri
Jul 23 00:05:32 2010
@@ -57,7 +57,8 @@ public final class WritableUtils {
}
}
- public static int writeCompressedByteArray(DataOutput out, byte[] bytes)
throws IOException {
+ public static int writeCompressedByteArray(DataOutput out,
+ byte[] bytes) throws IOException
{
if (bytes != null) {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
GZIPOutputStream gzout = new GZIPOutputStream(bos);
Modified:
hadoop/common/trunk/src/java/org/apache/hadoop/security/Credentials.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/security/Credentials.java?rev=966911&r1=966910&r2=966911&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/security/Credentials.java
(original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/security/Credentials.java
Fri Jul 23 00:05:32 2010
@@ -19,13 +19,17 @@
package org.apache.hadoop.security;
import java.io.DataInput;
+import java.io.DataInputStream;
import java.io.DataOutput;
+import java.io.DataOutputStream;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
@@ -114,24 +118,59 @@ public class Credentials implements Writ
}
/**
- * Convenience method for reading a file, and loading the Tokens
+ * Convenience method for reading a token storage file, and loading the
Tokens
* therein in the passed UGI
* @param filename
* @param conf
- * @param ugi
* @throws IOException
*/
- public static void readTokensAndLoadInUGI(String filename, Configuration
conf,
- UserGroupInformation ugi) throws IOException {
- Path localTokensFile = new Path (filename);
- FileSystem localFS = FileSystem.getLocal(conf);
- FSDataInputStream in = localFS.open(localTokensFile);
- Credentials ts = new Credentials();
- ts.readFields(in);
- for (Token<? extends TokenIdentifier> token : ts.getAllTokens()) {
- ugi.addToken(token);
+ public void readTokenStorageFile(Path filename,
+ Configuration conf) throws IOException {
+ FSDataInputStream in = filename.getFileSystem(conf).open(filename);
+ try {
+ readTokenStorageStream(in);
+ } catch(IOException ioe) {
+ throw new IOException("Exception reading " + filename, ioe);
+ } finally {
+ in.close();
}
}
+
+ /**
+ * Convenience method for reading a token storage file directly from a
+ * datainputstream
+ */
+ public void readTokenStorageStream(DataInputStream in) throws IOException {
+ byte[] magic = new byte[TOKEN_STORAGE_MAGIC.length];
+ in.readFully(magic);
+ if (!Arrays.equals(magic, TOKEN_STORAGE_MAGIC)) {
+ throw new IOException("Bad header found in token storage.");
+ }
+ byte version = in.readByte();
+ if (version != TOKEN_STORAGE_VERSION) {
+ throw new IOException("Unknown version " + version +
+ " in token storage.");
+ }
+ readFields(in);
+ }
+
+ private static final byte[] TOKEN_STORAGE_MAGIC = "HDTS".getBytes();
+ private static final byte TOKEN_STORAGE_VERSION = 0;
+
+ public void writeTokenStorageToStream(DataOutputStream os)
+ throws IOException {
+ os.write(TOKEN_STORAGE_MAGIC);
+ os.write(TOKEN_STORAGE_VERSION);
+ write(os);
+ }
+
+ public void writeTokenStorageFile(Path filename,
+ Configuration conf) throws IOException {
+ FSDataOutputStream os = filename.getFileSystem(conf).create(filename);
+ writeTokenStorageToStream(os);
+ os.close();
+ }
+
/**
* Stores all the keys to DataOutput
* @param out
@@ -151,7 +190,8 @@ public class Credentials implements Writ
WritableUtils.writeVInt(out, secretKeysMap.size());
for(Map.Entry<Text, byte[]> e : secretKeysMap.entrySet()) {
e.getKey().write(out);
- WritableUtils.writeCompressedByteArray(out, e.getValue());
+ WritableUtils.writeVInt(out, e.getValue().length);
+ out.write(e.getValue());
}
}
@@ -178,8 +218,23 @@ public class Credentials implements Writ
for(int i=0; i<size; i++) {
Text alias = new Text();
alias.readFields(in);
- byte[] key = WritableUtils.readCompressedByteArray(in);
- secretKeysMap.put(alias, key);
+ int len = WritableUtils.readVInt(in);
+ byte[] value = new byte[len];
+ in.readFully(value);
+ secretKeysMap.put(alias, value);
+ }
+ }
+
+ /**
+ * Copy all of the credentials from one credential object into another.
+ * @param other the credentials to copy
+ */
+ public void addAll(Credentials other) {
+ for(Map.Entry<Text, byte[]> secret: other.secretKeysMap.entrySet()) {
+ secretKeysMap.put(secret.getKey(), secret.getValue());
+ }
+ for(Map.Entry<Text, Token<?>> token: other.tokenMap.entrySet()){
+ tokenMap.put(token.getKey(), token.getValue());
}
}
}
Modified:
hadoop/common/trunk/src/java/org/apache/hadoop/security/UserGroupInformation.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/security/UserGroupInformation.java?rev=966911&r1=966910&r2=966911&view=diff
==============================================================================
---
hadoop/common/trunk/src/java/org/apache/hadoop/security/UserGroupInformation.java
(original)
+++
hadoop/common/trunk/src/java/org/apache/hadoop/security/UserGroupInformation.java
Fri Jul 23 00:05:32 2010
@@ -50,6 +50,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
@@ -135,6 +136,9 @@ public class UserGroupInformation {
private static boolean useKerberos;
/** Server-side groups fetching service */
private static Groups groups;
+ /** The configuration to use */
+ private static Configuration conf;
+
public static final long MIN_TIME_BEFORE_RELOGIN = 10 * 60 * 1000L;
@@ -188,6 +192,7 @@ public class UserGroupInformation {
"configuration", ioe);
}
isInitialized = true;
+ UserGroupInformation.conf = conf;
}
/**
@@ -398,9 +403,15 @@ public class UserGroupInformation {
login.login();
loginUser.setLogin(login);
loginUser = new UserGroupInformation(login.getSubject());
- String tokenFile = System.getenv(HADOOP_TOKEN_FILE_LOCATION);
- if (tokenFile != null && isSecurityEnabled()) {
- Credentials.readTokensAndLoadInUGI(tokenFile, new Configuration(),
loginUser);
+ String fileLocation = System.getenv(HADOOP_TOKEN_FILE_LOCATION);
+ if (fileLocation != null && isSecurityEnabled()) {
+ // load the token storage file and put all of the tokens into the
+ // user.
+ Credentials cred = new Credentials();
+ cred.readTokenStorageFile(new Path("file:///" + fileLocation), conf);
+ for (Token<?> token: cred.getAllTokens()) {
+ loginUser.addToken(token);
+ }
}
} catch (LoginException le) {
throw new IOException("failure to login", le);
Modified:
hadoop/common/trunk/src/java/org/apache/hadoop/util/GenericOptionsParser.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/util/GenericOptionsParser.java?rev=966911&r1=966910&r2=966911&view=diff
==============================================================================
---
hadoop/common/trunk/src/java/org/apache/hadoop/util/GenericOptionsParser.java
(original)
+++
hadoop/common/trunk/src/java/org/apache/hadoop/util/GenericOptionsParser.java
Fri Jul 23 00:05:32 2010
@@ -317,7 +317,9 @@ public class GenericOptionsParser {
throw new FileNotFoundException("File "+fileName+" does not exist.");
}
LOG.debug("setting conf tokensFile: " + fileName);
- conf.set("tokenCacheFile", localFs.makeQualified(p).toString());
+ conf.set("mapreduce.job.credentials.json", localFs.makeQualified(p)
+ .toString());
+
}
}
Modified:
hadoop/common/trunk/src/test/core/org/apache/hadoop/util/TestGenericOptionsParser.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/test/core/org/apache/hadoop/util/TestGenericOptionsParser.java?rev=966911&r1=966910&r2=966911&view=diff
==============================================================================
---
hadoop/common/trunk/src/test/core/org/apache/hadoop/util/TestGenericOptionsParser.java
(original)
+++
hadoop/common/trunk/src/test/core/org/apache/hadoop/util/TestGenericOptionsParser.java
Fri Jul 23 00:05:32 2010
@@ -126,7 +126,7 @@ public class TestGenericOptionsParser ex
Path tmpPath = new Path(tmpFile.toString());
localFs.create(tmpPath);
new GenericOptionsParser(conf, args);
- String fileName = conf.get("tokenCacheFile");
+ String fileName = conf.get("mapreduce.job.credentials.json");
assertNotNull("files is null", fileName);
assertEquals("files option does not match",
localFs.makeQualified(tmpPath).toString(), fileName);
