Author: omalley Date: Thu Feb 25 14:14:18 2010 New Revision: 916290 URL: http://svn.apache.org/viewvc?rev=916290&view=rev Log: HADOOP-6579. Provide a mechanism for encoding/decoding Tokens from a url-safe string and change the commons-code library to 1.4. (omalley)
Modified: hadoop/common/trunk/.eclipse.templates/.classpath hadoop/common/trunk/CHANGES.txt hadoop/common/trunk/ivy/hadoop-core-template.xml hadoop/common/trunk/ivy/libraries.properties hadoop/common/trunk/src/java/org/apache/hadoop/security/token/Token.java hadoop/common/trunk/src/test/core/org/apache/hadoop/io/compress/TestCodec.java hadoop/common/trunk/src/test/core/org/apache/hadoop/security/token/TestToken.java Modified: hadoop/common/trunk/.eclipse.templates/.classpath URL: http://svn.apache.org/viewvc/hadoop/common/trunk/.eclipse.templates/.classpath?rev=916290&r1=916289&r2=916290&view=diff ============================================================================== --- hadoop/common/trunk/.eclipse.templates/.classpath (original) +++ hadoop/common/trunk/.eclipse.templates/.classpath Thu Feb 25 14:14:18 2010 @@ -7,7 +7,7 @@ <classpathentry kind="var" path="ANT_HOME/lib/ant.jar"/> <classpathentry kind="lib" path="build/ivy/lib/Hadoop-Core/common/avro-1.2.0.jar"/> <classpathentry kind="lib" path="build/ivy/lib/Hadoop-Core/common/commons-cli-1.2.jar"/> - <classpathentry kind="lib" path="build/ivy/lib/Hadoop-Core/common/commons-codec-1.3.jar"/> + <classpathentry kind="lib" path="build/ivy/lib/Hadoop-Core/common/commons-codec-1.4.jar"/> <classpathentry kind="lib" path="build/ivy/lib/Hadoop-Core/common/commons-el-1.0.jar"/> <classpathentry kind="lib" path="build/ivy/lib/Hadoop-Core/common/commons-httpclient-3.0.1.jar"/> <classpathentry kind="lib" path="build/ivy/lib/Hadoop-Core/common/commons-logging-1.1.1.jar"/> Modified: hadoop/common/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=916290&r1=916289&r2=916290&view=diff ============================================================================== --- hadoop/common/trunk/CHANGES.txt (original) +++ hadoop/common/trunk/CHANGES.txt Thu Feb 25 14:14:18 2010 @@ -161,6 +161,9 @@ HADOOP-6543. Allows secure clients to talk to unsecure clusters. (Kan Zhang via ddas) + HADOOP-6579. Provide a mechanism for encoding/decoding Tokens from + a url-safe string and change the commons-code library to 1.4. (omalley) + OPTIMIZATIONS HADOOP-6467. Improve the performance on HarFileSystem.listStatus(..). Modified: hadoop/common/trunk/ivy/hadoop-core-template.xml URL: http://svn.apache.org/viewvc/hadoop/common/trunk/ivy/hadoop-core-template.xml?rev=916290&r1=916289&r2=916290&view=diff ============================================================================== --- hadoop/common/trunk/ivy/hadoop-core-template.xml (original) +++ hadoop/common/trunk/ivy/hadoop-core-template.xml Thu Feb 25 14:14:18 2010 @@ -41,7 +41,7 @@ <dependency> <groupId>commons-codec</groupId> <artifactId>commons-codec</artifactId> - <version>1.3</version> + <version>1.4</version> </dependency> <dependency> <groupId>commons-net</groupId> Modified: hadoop/common/trunk/ivy/libraries.properties URL: http://svn.apache.org/viewvc/hadoop/common/trunk/ivy/libraries.properties?rev=916290&r1=916289&r2=916290&view=diff ============================================================================== --- hadoop/common/trunk/ivy/libraries.properties (original) +++ hadoop/common/trunk/ivy/libraries.properties Thu Feb 25 14:14:18 2010 @@ -23,7 +23,7 @@ commons-cli.version=1.2 commons-cli2.version=2.0-mahout -commons-codec.version=1.3 +commons-codec.version=1.4 commons-collections.version=3.1 commons-httpclient.version=3.0.1 commons-lang.version=2.4 Modified: hadoop/common/trunk/src/java/org/apache/hadoop/security/token/Token.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/security/token/Token.java?rev=916290&r1=916289&r2=916290&view=diff ============================================================================== --- hadoop/common/trunk/src/java/org/apache/hadoop/security/token/Token.java (original) +++ hadoop/common/trunk/src/java/org/apache/hadoop/security/token/Token.java Thu Feb 25 14:14:18 2010 @@ -21,9 +21,15 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.util.Arrays; +import org.apache.commons.codec.binary.Base64; + +import org.apache.hadoop.io.DataInputBuffer; +import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparator; import org.apache.hadoop.io.WritableUtils; /** @@ -47,7 +53,21 @@ kind = id.getKind(); service = new Text(); } - + + /** + * Construct a token from the components. + * @param identifier the token identifier + * @param password the token's password + * @param kind the kind of token + * @param service the service for this token + */ + public Token(byte[] identifier, byte[] password, Text kind, Text service) { + this.identifier = identifier; + this.password = password; + this.kind = kind; + this.service = service; + } + /** * Default constructor */ @@ -123,4 +143,103 @@ kind.write(out); service.write(out); } + + /** + * Generate a string with the url-quoted base64 encoded serialized form + * of the Writable. + * @param obj the object to serialize + * @return the encoded string + * @throws IOException + */ + private static String encodeWritable(Writable obj) throws IOException { + DataOutputBuffer buf = new DataOutputBuffer(); + obj.write(buf); + Base64 encoder = new Base64(0, null, true); + byte[] raw = new byte[buf.getLength()]; + System.arraycopy(buf.getData(), 0, raw, 0, buf.getLength()); + return encoder.encodeToString(raw); + } + + /** + * Modify the writable to the value from the newValue + * @param obj the object to read into + * @param newValue the string with the url-safe base64 encoded bytes + * @throws IOException + */ + private static void decodeWritable(Writable obj, + String newValue) throws IOException { + Base64 decoder = new Base64(0, null, true); + DataInputBuffer buf = new DataInputBuffer(); + byte[] decoded = decoder.decode(newValue); + buf.reset(decoded, decoded.length); + obj.readFields(buf); + } + + /** + * Encode this token as a url safe string + * @return the encoded string + * @throws IOException + */ + public String encodeToUrlString() throws IOException { + return encodeWritable(this); + } + + /** + * Decode the given url safe string into this token. + * @param newValue the encoded string + * @throws IOException + */ + public void decodeFromUrlString(String newValue) throws IOException { + decodeWritable(this, newValue); + } + + @SuppressWarnings("unchecked") + @Override + public boolean equals(Object right) { + if (this == right) { + return true; + } else if (right == null || getClass() != right.getClass()) { + return false; + } else { + Token<T> r = (Token<T>) right; + return Arrays.equals(identifier, r.identifier) && + Arrays.equals(password, r.password) && + kind.equals(r.kind) && + service.equals(r.service); + } + } + + @Override + public int hashCode() { + return WritableComparator.hashBytes(identifier, identifier.length); + } + + private static void addBinaryBuffer(StringBuilder buffer, byte[] bytes) { + for (int idx = 0; idx < bytes.length; idx++) { + // if not the first, put a blank separator in + if (idx != 0) { + buffer.append(' '); + } + String num = Integer.toHexString(0xff & bytes[idx]); + // if it is only one digit, add a leading 0. + if (num.length() < 2) { + buffer.append('0'); + } + buffer.append(num); + } + } + + @Override + public String toString() { + StringBuilder buffer = new StringBuilder(); + buffer.append("Ident: "); + addBinaryBuffer(buffer, identifier); + buffer.append(", Pass: "); + addBinaryBuffer(buffer, password); + buffer.append(", Kind: "); + buffer.append(kind.toString()); + buffer.append(", Service: "); + buffer.append(service.toString()); + return buffer.toString(); + } } Modified: hadoop/common/trunk/src/test/core/org/apache/hadoop/io/compress/TestCodec.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/test/core/org/apache/hadoop/io/compress/TestCodec.java?rev=916290&r1=916289&r2=916290&view=diff ============================================================================== --- hadoop/common/trunk/src/test/core/org/apache/hadoop/io/compress/TestCodec.java (original) +++ hadoop/common/trunk/src/test/core/org/apache/hadoop/io/compress/TestCodec.java Thu Feb 25 14:14:18 2010 @@ -245,7 +245,7 @@ codec.getClass().getSimpleName()); final Path file = new Path(wd, "test" + codec.getDefaultExtension()); final byte[] b = new byte[REC_SIZE]; - final Base64 b64 = new Base64(); + final Base64 b64 = new Base64(0, null); DataOutputStream fout = null; Compressor cmp = CodecPool.getCompressor(codec); try { Modified: hadoop/common/trunk/src/test/core/org/apache/hadoop/security/token/TestToken.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/test/core/org/apache/hadoop/security/token/TestToken.java?rev=916290&r1=916289&r2=916290&view=diff ============================================================================== --- hadoop/common/trunk/src/test/core/org/apache/hadoop/security/token/TestToken.java (original) +++ hadoop/common/trunk/src/test/core/org/apache/hadoop/security/token/TestToken.java Thu Feb 25 14:14:18 2010 @@ -22,6 +22,7 @@ import java.util.Arrays; import org.apache.hadoop.io.*; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; import junit.framework.TestCase; @@ -58,4 +59,40 @@ destToken.readFields(in); assertTrue(checkEqual(sourceToken, destToken)); } + + private static void checkUrlSafe(String str) throws Exception { + int len = str.length(); + for(int i=0; i < len; ++i) { + char ch = str.charAt(i); + if (ch == '-') continue; + if (ch == '_') continue; + if (ch >= '0' && ch <= '9') continue; + if (ch >= 'A' && ch <= 'Z') continue; + if (ch >= 'a' && ch <= 'z') continue; + fail("Encoded string " + str + + " has invalid character at position " + i); + } + } + + public static void testEncodeWritable() throws Exception { + String[] values = new String[]{"", "a", "bb", "ccc", "dddd", "eeeee", + "ffffff", "ggggggg", "hhhhhhhh", "iiiiiiiii", + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLM" + + "nopqrstuvwxyz012345678...@#$%^&*()-=_+[]{}|;':,./<>?"}; + Token<AbstractDelegationTokenIdentifier> orig; + Token<AbstractDelegationTokenIdentifier> copy = + new Token<AbstractDelegationTokenIdentifier>(); + // ensure that for each string the input and output values match + for(int i=0; i< values.length; ++i) { + String val = values[i]; + System.out.println("Input = " + val); + orig = new Token<AbstractDelegationTokenIdentifier>(val.getBytes(), + val.getBytes(), new Text(val), new Text(val)); + String encode = orig.encodeToUrlString(); + copy.decodeFromUrlString(encode); + assertEquals(orig, copy); + checkUrlSafe(encode); + } + } + }