Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java?rev=1337645&r1=1337644&r2=1337645&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java Sat May 12 20:52:34 2012 @@ -18,10 +18,15 @@ package org.apache.hadoop.security.token; +import com.google.common.collect.Maps; + +import java.io.ByteArrayInputStream; import java.io.DataInput; +import java.io.DataInputStream; import java.io.DataOutput; import java.io.IOException; import java.util.Arrays; +import java.util.Map; import java.util.ServiceLoader; import org.apache.commons.codec.binary.Base64; @@ -37,6 +42,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparator; import org.apache.hadoop.io.WritableUtils; +import org.apache.hadoop.util.ReflectionUtils; /** * The client-side form of the token. @@ -45,6 +51,9 @@ import org.apache.hadoop.io.WritableUtil @InterfaceStability.Evolving public class Token<T extends TokenIdentifier> implements Writable { public static final Log LOG = LogFactory.getLog(Token.class); + + private static Map<Text, Class<? extends TokenIdentifier>> tokenKindMap; + private byte[] identifier; private byte[] password; private Text kind; @@ -100,13 +109,49 @@ public class Token<T extends TokenIdenti } /** - * Get the token identifier - * @return the token identifier + * Get the token identifier's byte representation + * @return the token identifier's byte representation */ public byte[] getIdentifier() { return identifier; } + private static synchronized Class<? extends TokenIdentifier> + getClassForIdentifier(Text kind) { + if (tokenKindMap == null) { + tokenKindMap = Maps.newHashMap(); + for (TokenIdentifier id : ServiceLoader.load(TokenIdentifier.class)) { + tokenKindMap.put(id.getKind(), id.getClass()); + } + } + Class<? extends TokenIdentifier> cls = tokenKindMap.get(kind); + if (cls == null) { + LOG.warn("Cannot find class for token kind " + kind); + return null; + } + return cls; + } + + /** + * Get the token identifier object, or null if it could not be constructed + * (because the class could not be loaded, for example). + * @return the token identifier, or null + * @throws IOException + */ + @SuppressWarnings("unchecked") + public T decodeIdentifier() throws IOException { + Class<? extends TokenIdentifier> cls = getClassForIdentifier(getKind()); + if (cls == null) { + return null; + } + TokenIdentifier tokenIdentifier = ReflectionUtils.newInstance(cls, null); + ByteArrayInputStream buf = new ByteArrayInputStream(identifier); + DataInputStream in = new DataInputStream(buf); + tokenIdentifier.readFields(in); + in.close(); + return (T) tokenIdentifier; + } + /** * Get the token password/secret * @return the token password/secret @@ -260,16 +305,31 @@ public class Token<T extends TokenIdenti buffer.append(num); } } + + private void identifierToString(StringBuilder buffer) { + T id = null; + try { + id = decodeIdentifier(); + } catch (IOException e) { + // handle in the finally block + } finally { + if (id != null) { + buffer.append("(").append(id).append(")"); + } else { + addBinaryBuffer(buffer, identifier); + } + } + } @Override public String toString() { StringBuilder buffer = new StringBuilder(); - buffer.append("Ident: "); - addBinaryBuffer(buffer, identifier); - buffer.append(", Kind: "); + buffer.append("Kind: "); buffer.append(kind.toString()); buffer.append(", Service: "); buffer.append(service.toString()); + buffer.append(", Ident: "); + identifierToString(buffer); return buffer.toString(); }
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionUtil.java?rev=1337645&r1=1337644&r2=1337645&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionUtil.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionUtil.java Sat May 12 20:52:34 2012 @@ -22,12 +22,21 @@ import java.util.regex.Pattern; import org.apache.hadoop.classification.InterfaceAudience; +import com.google.common.collect.ComparisonChain; + @InterfaceAudience.Private public abstract class VersionUtil { private static final Pattern COMPONENT_GROUPS = Pattern.compile("(\\d+)|(\\D+)"); /** + * Suffix added by maven for nightly builds and other snapshot releases. + * These releases are considered to precede the non-SNAPSHOT version + * with the same version number. + */ + private static final String SNAPSHOT_SUFFIX = "-SNAPSHOT"; + + /** * This function splits the two versions on "." and performs a * naturally-ordered comparison of the resulting components. For example, the * version string "0.3" is considered to precede "0.20", despite the fact that @@ -48,6 +57,11 @@ public abstract class VersionUtil { * between the two versions, then the version with fewer components is * considered to precede the version with more components. * + * In addition to the above rules, there is one special case: maven SNAPSHOT + * releases are considered to precede a non-SNAPSHOT release with an + * otherwise identical version number. For example, 2.0-SNAPSHOT precedes + * 2.0. + * * This function returns a negative integer if version1 precedes version2, a * positive integer if version2 precedes version1, and 0 if and only if the * two versions' components are identical in value and cardinality. @@ -61,6 +75,11 @@ public abstract class VersionUtil { * versions are equal. */ public static int compareVersions(String version1, String version2) { + boolean isSnapshot1 = version1.endsWith(SNAPSHOT_SUFFIX); + boolean isSnapshot2 = version2.endsWith(SNAPSHOT_SUFFIX); + version1 = stripSnapshotSuffix(version1); + version2 = stripSnapshotSuffix(version2); + String[] version1Parts = version1.split("\\."); String[] version2Parts = version2.split("\\."); @@ -87,9 +106,21 @@ public abstract class VersionUtil { return component1.length() - component2.length(); } } - return version1Parts.length - version2Parts.length; + + return ComparisonChain.start() + .compare(version1Parts.length, version2Parts.length) + .compare(isSnapshot2, isSnapshot1) + .result(); } + private static String stripSnapshotSuffix(String version) { + if (version.endsWith(SNAPSHOT_SUFFIX)) { + return version.substring(0, version.length() - SNAPSHOT_SUFFIX.length()); + } else { + return version; + } + } + private static boolean isNumeric(String s) { try { Integer.parseInt(s); Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/native/acinclude.m4 URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/native/acinclude.m4?rev=1337645&r1=1337644&r2=1337645&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/native/acinclude.m4 (original) +++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/native/acinclude.m4 Sat May 12 20:52:34 2012 @@ -1,4 +1,4 @@ -# AC_COMPUTE_NEEDED_DSO(LIBRARY, PREPROC_SYMBOL) +# AC_COMPUTE_NEEDED_DSO(LIBRARY, TEST_PROGRAM, PREPROC_SYMBOL) # -------------------------------------------------- # Compute the 'actual' dynamic-library used # for LIBRARY and set it to PREPROC_SYMBOL @@ -6,7 +6,7 @@ AC_DEFUN([AC_COMPUTE_NEEDED_DSO], [ AC_CACHE_CHECK([Checking for the 'actual' dynamic-library for '-l$1'], ac_cv_libname_$1, [ - echo 'int main(int argc, char **argv){return 0;}' > conftest.c + echo '$2' > conftest.c if test -z "`${CC} ${LDFLAGS} -o conftest conftest.c -l$1 2>&1`"; then dnl Try objdump and ldd in that order to get the dynamic library if test ! -z "`which objdump | grep -v 'no objdump'`"; then @@ -24,5 +24,5 @@ AC_CACHE_CHECK([Checking for the 'actual rm -f conftest* ] ) -AC_DEFINE_UNQUOTED($2, ${ac_cv_libname_$1}, [The 'actual' dynamic-library for '-l$1']) +AC_DEFINE_UNQUOTED($3, ${ac_cv_libname_$1}, [The 'actual' dynamic-library for '-l$1']) ])# AC_COMPUTE_NEEDED_DSO Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/native/configure.ac URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/native/configure.ac?rev=1337645&r1=1337644&r2=1337645&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/native/configure.ac (original) +++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/native/configure.ac Sat May 12 20:52:34 2012 @@ -87,10 +87,20 @@ CPPFLAGS=$cppflags_bak AC_SUBST([JNI_CPPFLAGS]) dnl Check for zlib headers -AC_CHECK_HEADERS([zlib.h zconf.h], AC_COMPUTE_NEEDED_DSO(z,HADOOP_ZLIB_LIBRARY), AC_MSG_ERROR(Zlib headers were not found... native-hadoop library needs zlib to build. Please install the requisite zlib development package.)) +AC_CHECK_HEADERS([zlib.h zconf.h], + AC_COMPUTE_NEEDED_DSO(z, + [#include "zlib.h" + int main(int argc, char **argv){zlibVersion();return 0;}], + HADOOP_ZLIB_LIBRARY), + AC_MSG_ERROR(Zlib headers were not found... native-hadoop library needs zlib to build. Please install the requisite zlib development package.)) dnl Check for snappy headers -AC_CHECK_HEADERS([snappy-c.h], AC_COMPUTE_NEEDED_DSO(snappy,HADOOP_SNAPPY_LIBRARY), AC_MSG_WARN(Snappy headers were not found... building without snappy.)) +AC_CHECK_HEADERS([snappy-c.h], + AC_COMPUTE_NEEDED_DSO(snappy, + [#include "snappy-c.h" + int main(int argc, char **argv){snappy_compress(0,0,0,0);return 0;}], + HADOOP_SNAPPY_LIBRARY), + AC_MSG_WARN(Snappy headers were not found... building without snappy.)) dnl Check for headers needed by the native Group resolution implementation AC_CHECK_HEADERS([fcntl.h stdlib.h string.h unistd.h], [], AC_MSG_ERROR(Some system headers not found... please ensure their presence on your platform.)) Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c?rev=1337645&r1=1337644&r2=1337645&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c (original) +++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c Sat May 12 20:52:34 2012 @@ -70,7 +70,7 @@ Java_org_apache_hadoop_security_JniBased // set the name of the group for subsequent calls to getnetgrent // note that we want to end group lokup regardless whether setnetgrent - // was successfull or not (as long as it was called we need to call + // was successful or not (as long as it was called we need to call // endnetgrent) setnetgrentCalledFlag = 1; if(setnetgrent(cgroup) == 1) { Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hadoop-env.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hadoop-env.sh?rev=1337645&r1=1337644&r2=1337645&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hadoop-env.sh (original) +++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hadoop-env.sh Sat May 12 20:52:34 2012 @@ -48,10 +48,10 @@ done export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true $HADOOP_CLIENT_OPTS" # Command specific options appended to HADOOP_OPTS when specified -export HADOOP_NAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=INFO,RFAAUDIT $HADOOP_NAMENODE_OPTS" +export HADOOP_NAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_NAMENODE_OPTS" export HADOOP_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS $HADOOP_DATANODE_OPTS" -export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=INFO,RFAAUDIT $HADOOP_SECONDARYNAMENODE_OPTS" +export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_SECONDARYNAMENODE_OPTS" # The ZKFC does not need a large heap, and keeping it small avoids # any potential for long GC pauses Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hdfs-site.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hdfs-site.xml?rev=1337645&r1=1337644&r2=1337645&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hdfs-site.xml (original) +++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hdfs-site.xml Sat May 12 20:52:34 2012 @@ -129,13 +129,6 @@ </property> <property> - <name>dfs.secondary.https.port</name> - <value>50490</value> - <description>The https port where secondary-namenode binds</description> - - </property> - - <property> <name>dfs.datanode.kerberos.principal</name> <value>dn/_HOST@${local.realm}</value> <description> Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/log4j.properties URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/log4j.properties?rev=1337645&r1=1337644&r2=1337645&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/log4j.properties (original) +++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/log4j.properties Sat May 12 20:52:34 2012 @@ -102,7 +102,7 @@ log4j.appender.TLA.layout.ConversionPatt # #Security appender # -hadoop.security.logger=INFO,console +hadoop.security.logger=INFO,NullAppender hadoop.security.log.maxfilesize=256MB hadoop.security.log.maxbackupindex=20 log4j.category.SecurityLogger=${hadoop.security.logger} @@ -126,7 +126,7 @@ log4j.appender.DRFAS.DatePattern=.yyyy-M # # hdfs audit logging # -hdfs.audit.logger=INFO,console +hdfs.audit.logger=INFO,NullAppender hdfs.audit.log.maxfilesize=256MB hdfs.audit.log.maxbackupindex=20 log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger} @@ -141,7 +141,7 @@ log4j.appender.RFAAUDIT.MaxBackupIndex=$ # # mapred audit logging # -mapred.audit.logger=INFO,console +mapred.audit.logger=INFO,NullAppender mapred.audit.log.maxfilesize=256MB mapred.audit.log.maxbackupindex=20 log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger} Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/proto/RpcPayloadHeader.proto URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/proto/RpcPayloadHeader.proto?rev=1337645&r1=1337644&r2=1337645&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/proto/RpcPayloadHeader.proto (original) +++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/proto/RpcPayloadHeader.proto Sat May 12 20:52:34 2012 @@ -19,7 +19,6 @@ option java_package = "org.apache.hadoop option java_outer_classname = "RpcPayloadHeaderProtos"; option java_generate_equals_and_hash = true; - /** * This is the rpc payload header. It is sent with every rpc call. * @@ -34,8 +33,6 @@ option java_generate_equals_and_hash = t * */ - - /** * RpcKind determine the rpcEngine and the serialization of the rpc payload */ @@ -54,5 +51,27 @@ enum RpcPayloadOperationProto { message RpcPayloadHeaderProto { // the header for the RpcRequest optional RpcKindProto rpcKind = 1; optional RpcPayloadOperationProto rpcOp = 2; - optional uint32 callId = 3; // each rpc has a callId that is also used in response + required uint32 callId = 3; // each rpc has a callId that is also used in response +} + +enum RpcStatusProto { + SUCCESS = 0; // RPC succeeded + ERROR = 1; // RPC Failed + FATAL = 2; // Fatal error - connection is closed +} + +/** + * Rpc Response Header + * - If successfull then the Respose follows after this header + * - length (4 byte int), followed by the response + * - If error or fatal - the exception info follow + * - length (4 byte int) Class name of exception - UTF-8 string + * - length (4 byte int) Stacktrace - UTF-8 string + * - if the strings are null then the length is -1 + * In case of Fatal error then the respose contains the Serverside's IPC version + */ +message RpcResponseHeaderProto { + required uint32 callId = 1; // callId used in Request + required RpcStatusProto status = 2; + optional uint32 serverIpcVersionNum = 3; // in case of an fatal IPC error } Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml?rev=1337645&r1=1337644&r2=1337645&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml (original) +++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml Sat May 12 20:52:34 2012 @@ -63,6 +63,15 @@ </property> <property> + <name>hadoop.security.instrumentation.requires.admin</name> + <value>false</value> + <description> + Indicates if administrator ACLs are required to access + instrumentation servlets (JMX, METRICS, CONF, STACKS). + </description> +</property> + +<property> <name>hadoop.security.authentication</name> <value>simple</value> <description>Possible values are simple (no authentication), and kerberos Propchange: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/core/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1333291-1337618 Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java?rev=1337645&r1=1337644&r2=1337645&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java Sat May 12 20:52:34 2012 @@ -18,11 +18,14 @@ package org.apache.hadoop.fs; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem.Statistics; + import static org.apache.hadoop.fs.FileSystemTestHelper.*; import java.io.*; import static org.junit.Assert.*; + import org.junit.Before; import org.junit.Test; @@ -233,4 +236,16 @@ public class TestLocalFileSystem { assertTrue("Did not delete file", fs.delete(file1)); assertTrue("Did not delete non-empty dir", fs.delete(dir1)); } + + @Test + public void testStatistics() throws Exception { + FileSystem.getLocal(new Configuration()); + int fileSchemeCount = 0; + for (Statistics stats : FileSystem.getAllStatistics()) { + if (stats.getScheme().equals("file")) { + fileSchemeCount++; + } + } + assertEquals(1, fileSchemeCount); + } } Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java?rev=1337645&r1=1337644&r2=1337645&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java Sat May 12 20:52:34 2012 @@ -71,11 +71,8 @@ public class ViewFileSystemBaseTest { @Before public void setUp() throws Exception { - targetTestRoot = FileSystemTestHelper.getAbsoluteTestRootPath(fsTarget); - // In case previous test was killed before cleanup - fsTarget.delete(targetTestRoot, true); + initializeTargetTestRoot(); - fsTarget.mkdirs(targetTestRoot); // Make user and data dirs - we creates links to them in the mount table fsTarget.mkdirs(new Path(targetTestRoot,"user")); fsTarget.mkdirs(new Path(targetTestRoot,"data")); @@ -99,7 +96,16 @@ public class ViewFileSystemBaseTest { fsTarget.delete(FileSystemTestHelper.getTestRootPath(fsTarget), true); } + void initializeTargetTestRoot() throws IOException { + targetTestRoot = FileSystemTestHelper.getAbsoluteTestRootPath(fsTarget); + // In case previous test was killed before cleanup + fsTarget.delete(targetTestRoot, true); + + fsTarget.mkdirs(targetTestRoot); + } + void setupMountPoints() { + ConfigUtil.addLink(conf, "/targetRoot", targetTestRoot.toUri()); ConfigUtil.addLink(conf, "/user", new Path(targetTestRoot,"user").toUri()); ConfigUtil.addLink(conf, "/user2", new Path(targetTestRoot,"user").toUri()); ConfigUtil.addLink(conf, "/data", new Path(targetTestRoot,"data").toUri()); @@ -121,7 +127,7 @@ public class ViewFileSystemBaseTest { } int getExpectedMountPoints() { - return 7; + return 8; } /** @@ -166,7 +172,7 @@ public class ViewFileSystemBaseTest { } } } - Assert.assertEquals(expectedTokenCount / 2, delTokens.size()); + Assert.assertEquals((expectedTokenCount + 1) / 2, delTokens.size()); } int getExpectedDelegationTokenCountWithCredentials() { @@ -309,6 +315,16 @@ public class ViewFileSystemBaseTest { Assert.assertTrue("Renamed dest should exist as dir in target", fsTarget.isDirectory(new Path(targetTestRoot,"user/dirFooBar"))); + // Make a directory under a directory that's mounted from the root of another FS + fsView.mkdirs(new Path("/targetRoot/dirFoo")); + Assert.assertTrue(fsView.exists(new Path("/targetRoot/dirFoo"))); + boolean dirFooPresent = false; + for (FileStatus fileStatus : fsView.listStatus(new Path("/targetRoot/"))) { + if (fileStatus.getPath().getName().equals("dirFoo")) { + dirFooPresent = true; + } + } + Assert.assertTrue(dirFooPresent); } // rename across mount points that point to same target also fail @@ -418,7 +434,7 @@ public class ViewFileSystemBaseTest { } int getExpectedDirPaths() { - return 6; + return 7; } @Test Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java?rev=1337645&r1=1337644&r2=1337645&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java Sat May 12 20:52:34 2012 @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.AbstractFile import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileContextTestHelper; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.FileContextTestHelper.fileType; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FsConstants; @@ -77,12 +78,8 @@ public class ViewFsBaseTest { @Before public void setUp() throws Exception { - - targetTestRoot = FileContextTestHelper.getAbsoluteTestRootPath(fcTarget); - // In case previous test was killed before cleanup - fcTarget.delete(targetTestRoot, true); + initializeTargetTestRoot(); - fcTarget.mkdir(targetTestRoot, FileContext.DEFAULT_PERM, true); // Make user and data dirs - we creates links to them in the mount table fcTarget.mkdir(new Path(targetTestRoot,"user"), FileContext.DEFAULT_PERM, true); @@ -100,6 +97,7 @@ public class ViewFsBaseTest { // Set up the defaultMT in the config with our mount point links conf = new Configuration(); + ConfigUtil.addLink(conf, "/targetRoot", targetTestRoot.toUri()); ConfigUtil.addLink(conf, "/user", new Path(targetTestRoot,"user").toUri()); ConfigUtil.addLink(conf, "/user2", @@ -118,6 +116,14 @@ public class ViewFsBaseTest { fcView = FileContext.getFileContext(FsConstants.VIEWFS_URI, conf); // Also try viewfs://default/ - note authority is name of mount table } + + void initializeTargetTestRoot() throws IOException { + targetTestRoot = FileContextTestHelper.getAbsoluteTestRootPath(fcTarget); + // In case previous test was killed before cleanup + fcTarget.delete(targetTestRoot, true); + + fcTarget.mkdir(targetTestRoot, FileContext.DEFAULT_PERM, true); + } @After public void tearDown() throws Exception { @@ -128,7 +134,11 @@ public class ViewFsBaseTest { public void testGetMountPoints() { ViewFs viewfs = (ViewFs) fcView.getDefaultFileSystem(); MountPoint[] mountPoints = viewfs.getMountPoints(); - Assert.assertEquals(7, mountPoints.length); + Assert.assertEquals(8, mountPoints.length); + } + + int getExpectedDelegationTokenCount() { + return 0; } /** @@ -140,7 +150,7 @@ public class ViewFsBaseTest { public void testGetDelegationTokens() throws IOException { List<Token<?>> delTokens = fcView.getDelegationTokens(new Path("/"), "sanjay"); - Assert.assertEquals(0, delTokens.size()); + Assert.assertEquals(getExpectedDelegationTokenCount(), delTokens.size()); } @@ -281,6 +291,19 @@ public class ViewFsBaseTest { Assert.assertTrue("Renamed dest should exist as dir in target", isDir(fcTarget,new Path(targetTestRoot,"user/dirFooBar"))); + // Make a directory under a directory that's mounted from the root of another FS + fcView.mkdir(new Path("/targetRoot/dirFoo"), FileContext.DEFAULT_PERM, false); + Assert.assertTrue(exists(fcView, new Path("/targetRoot/dirFoo"))); + boolean dirFooPresent = false; + RemoteIterator<FileStatus> dirContents = fcView.listStatus(new Path( + "/targetRoot/")); + while (dirContents.hasNext()) { + FileStatus fileStatus = dirContents.next(); + if (fileStatus.getPath().getName().equals("dirFoo")) { + dirFooPresent = true; + } + } + Assert.assertTrue(dirFooPresent); } // rename across mount points that point to same target also fail @@ -358,7 +381,7 @@ public class ViewFsBaseTest { FileStatus[] dirPaths = fcView.util().listStatus(new Path("/")); FileStatus fs; - Assert.assertEquals(6, dirPaths.length); + Assert.assertEquals(7, dirPaths.length); fs = FileContextTestHelper.containsPath(fcView, "/user", dirPaths); Assert.assertNotNull(fs); Assert.assertTrue("A mount should appear as symlink", fs.isSymlink()); Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java?rev=1337645&r1=1337644&r2=1337645&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java Sat May 12 20:52:34 2012 @@ -19,6 +19,7 @@ package org.apache.hadoop.http; +import org.apache.hadoop.security.authorize.AccessControlList; import org.junit.Assert; import org.apache.hadoop.conf.Configuration; @@ -70,6 +71,12 @@ public class HttpServerFunctionalTest ex return createServer(TEST, conf); } + public static HttpServer createTestServer(Configuration conf, AccessControlList adminsAcl) + throws IOException { + prepareTestWebapp(); + return createServer(TEST, conf, adminsAcl); + } + /** * Create but do not start the test webapp server. The test webapp dir is * prepared/checked in advance. @@ -132,6 +139,11 @@ public class HttpServerFunctionalTest ex throws IOException { return new HttpServer(webapp, "0.0.0.0", 0, true, conf); } + + public static HttpServer createServer(String webapp, Configuration conf, AccessControlList adminsAcl) + throws IOException { + return new HttpServer(webapp, "0.0.0.0", 0, true, conf, adminsAcl); + } /** * Create an HttpServer instance for the given webapp * @param webapp the webapp to work with Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java?rev=1337645&r1=1337644&r2=1337645&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java Sat May 12 20:52:34 2012 @@ -60,7 +60,6 @@ import org.apache.hadoop.security.author import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; -import org.mockito.Mock; import org.mockito.Mockito; import org.mortbay.util.ajax.JSON; @@ -360,6 +359,8 @@ public class TestHttpServer extends Http Configuration conf = new Configuration(); conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, true); + conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_INSTRUMENTATION_REQUIRES_ADMIN, + true); conf.set(HttpServer.FILTER_INITIALIZER_PROPERTY, DummyFilterInitializer.class.getName()); @@ -468,6 +469,26 @@ public class TestHttpServer extends Http } + @Test + public void testRequiresAuthorizationAccess() throws Exception { + Configuration conf = new Configuration(); + ServletContext context = Mockito.mock(ServletContext.class); + Mockito.when(context.getAttribute(HttpServer.CONF_CONTEXT_ATTRIBUTE)).thenReturn(conf); + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + HttpServletResponse response = Mockito.mock(HttpServletResponse.class); + + //requires admin access to instrumentation, FALSE by default + Assert.assertTrue(HttpServer.isInstrumentationAccessAllowed(context, request, response)); + + //requires admin access to instrumentation, TRUE + conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_INSTRUMENTATION_REQUIRES_ADMIN, true); + conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, true); + AccessControlList acls = Mockito.mock(AccessControlList.class); + Mockito.when(acls.isUserAllowed(Mockito.<UserGroupInformation>any())).thenReturn(false); + Mockito.when(context.getAttribute(HttpServer.ADMINS_ACL)).thenReturn(acls); + Assert.assertFalse(HttpServer.isInstrumentationAccessAllowed(context, request, response)); + } + @Test public void testBindAddress() throws Exception { checkBindAddress("0.0.0.0", 0, false).stop(); // hang onto this one for a bit more testing Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java?rev=1337645&r1=1337644&r2=1337645&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java Sat May 12 20:52:34 2012 @@ -20,6 +20,7 @@ package org.apache.hadoop.io; import junit.framework.TestCase; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.CharacterCodingException; import java.util.Random; @@ -107,7 +108,6 @@ public class TestText extends TestCase { } } - public void testIO() throws Exception { DataOutputBuffer out = new DataOutputBuffer(); DataInputBuffer in = new DataInputBuffer(); @@ -136,6 +136,40 @@ public class TestText extends TestCase { assertTrue(before.equals(after2)); } } + + public void doTestLimitedIO(String str, int strLen) throws IOException { + DataOutputBuffer out = new DataOutputBuffer(); + DataInputBuffer in = new DataInputBuffer(); + + out.reset(); + try { + Text.writeString(out, str, strLen); + fail("expected writeString to fail when told to write a string " + + "that was too long! The string was '" + str + "'"); + } catch (IOException e) { + } + Text.writeString(out, str, strLen + 1); + + // test that it reads correctly + in.reset(out.getData(), out.getLength()); + in.mark(strLen); + String after; + try { + after = Text.readString(in, strLen); + fail("expected readString to fail when told to read a string " + + "that was too long! The string was '" + str + "'"); + } catch (IOException e) { + } + in.reset(); + after = Text.readString(in, strLen + 1); + assertTrue(str.equals(after)); + } + + public void testLimitedIO() throws Exception { + doTestLimitedIO("abcd", 4); + doTestLimitedIO("", 0); + doTestLimitedIO("1", 1); + } public void testCompare() throws Exception { DataOutputBuffer out1 = new DataOutputBuffer(); @@ -192,16 +226,6 @@ public class TestText extends TestCase { assertTrue(text.find("\u20ac", 5)==11); } - public void testClear() { - Text text = new Text(); - assertEquals("", text.toString()); - assertEquals(0, text.getBytes().length); - text = new Text("abcd\u20acbdcd\u20ac"); - text.clear(); - assertEquals("", text.toString()); - assertEquals(0, text.getBytes().length); - } - public void testFindAfterUpdatingContents() throws Exception { Text text = new Text("abcd"); text.set("a".getBytes()); Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java?rev=1337645&r1=1337644&r2=1337645&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java Sat May 12 20:52:34 2012 @@ -323,6 +323,29 @@ public class TestRPC { } @Test + public void testProxyAddress() throws Exception { + Server server = RPC.getServer(TestProtocol.class, + new TestImpl(), ADDRESS, 0, conf); + TestProtocol proxy = null; + + try { + server.start(); + InetSocketAddress addr = NetUtils.getConnectAddress(server); + + // create a client + proxy = (TestProtocol)RPC.getProxy( + TestProtocol.class, TestProtocol.versionID, addr, conf); + + assertEquals(addr, RPC.getServerAddress(proxy)); + } finally { + server.stop(); + if (proxy != null) { + RPC.stopProxy(proxy); + } + } + } + + @Test public void testSlowRpc() throws Exception { System.out.println("Testing Slow RPC"); // create a server with two handlers Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java?rev=1337645&r1=1337644&r2=1337645&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java Sat May 12 20:52:34 2012 @@ -25,11 +25,16 @@ import java.net.ConnectException; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.NetworkInterface; +import java.net.ServerSocket; import java.net.Socket; import java.net.SocketException; +import java.net.SocketTimeoutException; import java.net.URI; import java.net.UnknownHostException; +import java.util.Arrays; import java.util.Enumeration; +import java.util.List; +import java.util.concurrent.TimeUnit; import junit.framework.AssertionFailedError; @@ -37,7 +42,9 @@ import org.apache.commons.lang.StringUti import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.NetUtilsTestResolver; +import org.junit.Assume; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -51,6 +58,13 @@ public class TestNetUtils { private static final String LOCAL_PORT_NAME = Integer.toString(LOCAL_PORT); /** + * Some slop around expected times when making sure timeouts behave + * as expected. We assume that they will be accurate to within + * this threshold. + */ + static final long TIME_FUDGE_MILLIS = 200; + + /** * Test that we can't accidentally connect back to the connecting socket due * to a quirk in the TCP spec. * @@ -81,6 +95,79 @@ public class TestNetUtils { } } + @Test + public void testSocketReadTimeoutWithChannel() throws Exception { + doSocketReadTimeoutTest(true); + } + + @Test + public void testSocketReadTimeoutWithoutChannel() throws Exception { + doSocketReadTimeoutTest(false); + } + + + private void doSocketReadTimeoutTest(boolean withChannel) + throws IOException { + // Binding a ServerSocket is enough to accept connections. + // Rely on the backlog to accept for us. + ServerSocket ss = new ServerSocket(0); + + Socket s; + if (withChannel) { + s = NetUtils.getDefaultSocketFactory(new Configuration()) + .createSocket(); + Assume.assumeNotNull(s.getChannel()); + } else { + s = new Socket(); + assertNull(s.getChannel()); + } + + SocketInputWrapper stm = null; + try { + NetUtils.connect(s, ss.getLocalSocketAddress(), 1000); + + stm = NetUtils.getInputStream(s, 1000); + assertReadTimeout(stm, 1000); + + // Change timeout, make sure it applies. + stm.setTimeout(1); + assertReadTimeout(stm, 1); + + // If there is a channel, then setting the socket timeout + // should not matter. If there is not a channel, it will + // take effect. + s.setSoTimeout(1000); + if (withChannel) { + assertReadTimeout(stm, 1); + } else { + assertReadTimeout(stm, 1000); + } + } finally { + IOUtils.closeStream(stm); + IOUtils.closeSocket(s); + ss.close(); + } + } + + private void assertReadTimeout(SocketInputWrapper stm, int timeoutMillis) + throws IOException { + long st = System.nanoTime(); + try { + stm.read(); + fail("Didn't time out"); + } catch (SocketTimeoutException ste) { + assertTimeSince(st, timeoutMillis); + } + } + + private void assertTimeSince(long startNanos, int expectedMillis) { + long durationNano = System.nanoTime() - startNanos; + long millis = TimeUnit.MILLISECONDS.convert( + durationNano, TimeUnit.NANOSECONDS); + assertTrue("Expected " + expectedMillis + "ms, but took " + millis, + Math.abs(millis - expectedMillis) < TIME_FUDGE_MILLIS); + } + /** * Test for { * @throws UnknownHostException @link NetUtils#getLocalInetAddress(String) @@ -512,6 +599,26 @@ public class TestNetUtils { assertEquals("scheme://host.a.b/path", uri.toString()); } + /** + * Test for {@link NetUtils#normalizeHostNames} + */ + @Test + public void testNormalizeHostName() { + List<String> hosts = Arrays.asList(new String[] {"127.0.0.1", + "localhost", "3w.org", "UnknownHost"}); + List<String> normalizedHosts = NetUtils.normalizeHostNames(hosts); + // when ipaddress is normalized, same address is expected in return + assertEquals(normalizedHosts.get(0), hosts.get(0)); + // for normalizing a resolvable hostname, resolved ipaddress is expected in return + assertFalse(normalizedHosts.get(1).equals(hosts.get(1))); + assertEquals(normalizedHosts.get(1), hosts.get(0)); + // this address HADOOP-8372: when normalizing a valid resolvable hostname start with numeric, + // its ipaddress is expected to return + assertFalse(normalizedHosts.get(2).equals(hosts.get(2))); + // return the same hostname after normalizing a irresolvable hostname. + assertEquals(normalizedHosts.get(3), hosts.get(3)); + } + @Test public void testGetHostNameOfIP() { assertNull(NetUtils.getHostNameOfIP(null)); Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestSocketIOWithTimeout.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestSocketIOWithTimeout.java?rev=1337645&r1=1337644&r2=1337645&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestSocketIOWithTimeout.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestSocketIOWithTimeout.java Sat May 12 20:52:34 2012 @@ -19,6 +19,7 @@ package org.apache.hadoop.net; import java.io.IOException; import java.io.InputStream; +import java.io.InterruptedIOException; import java.io.OutputStream; import java.net.SocketTimeoutException; import java.nio.channels.Pipe; @@ -26,8 +27,13 @@ import java.util.Arrays; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.test.MultithreadedTestUtil; +import org.apache.hadoop.test.MultithreadedTestUtil.TestContext; +import org.apache.hadoop.test.MultithreadedTestUtil.TestingThread; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; /** * This tests timout out from SocketInputStream and @@ -36,14 +42,17 @@ import junit.framework.TestCase; * Normal read and write using these streams are tested by pretty much * every DFS unit test. */ -public class TestSocketIOWithTimeout extends TestCase { +public class TestSocketIOWithTimeout { static Log LOG = LogFactory.getLog(TestSocketIOWithTimeout.class); private static int TIMEOUT = 1*1000; private static String TEST_STRING = "1234567890"; + + private MultithreadedTestUtil.TestContext ctx = new TestContext(); - private void doIO(InputStream in, OutputStream out) throws IOException { + private void doIO(InputStream in, OutputStream out, + int expectedTimeout) throws IOException { /* Keep on writing or reading until we get SocketTimeoutException. * It expects this exception to occur within 100 millis of TIMEOUT. */ @@ -61,34 +70,15 @@ public class TestSocketIOWithTimeout ext long diff = System.currentTimeMillis() - start; LOG.info("Got SocketTimeoutException as expected after " + diff + " millis : " + e.getMessage()); - assertTrue(Math.abs(TIMEOUT - diff) <= 200); + assertTrue(Math.abs(expectedTimeout - diff) <= + TestNetUtils.TIME_FUDGE_MILLIS); break; } } } - /** - * Just reads one byte from the input stream. - */ - static class ReadRunnable implements Runnable { - private InputStream in; - - public ReadRunnable(InputStream in) { - this.in = in; - } - public void run() { - try { - in.read(); - } catch (IOException e) { - LOG.info("Got expection while reading as expected : " + - e.getMessage()); - return; - } - assertTrue(false); - } - } - - public void testSocketIOWithTimeout() throws IOException { + @Test + public void testSocketIOWithTimeout() throws Exception { // first open pipe: Pipe pipe = Pipe.open(); @@ -96,7 +86,7 @@ public class TestSocketIOWithTimeout ext Pipe.SinkChannel sink = pipe.sink(); try { - InputStream in = new SocketInputStream(source, TIMEOUT); + final InputStream in = new SocketInputStream(source, TIMEOUT); OutputStream out = new SocketOutputStream(sink, TIMEOUT); byte[] writeBytes = TEST_STRING.getBytes(); @@ -105,37 +95,62 @@ public class TestSocketIOWithTimeout ext out.write(writeBytes); out.write(byteWithHighBit); - doIO(null, out); + doIO(null, out, TIMEOUT); in.read(readBytes); assertTrue(Arrays.equals(writeBytes, readBytes)); assertEquals(byteWithHighBit & 0xff, in.read()); - doIO(in, null); + doIO(in, null, TIMEOUT); + + // Change timeout on the read side. + ((SocketInputStream)in).setTimeout(TIMEOUT * 2); + doIO(in, null, TIMEOUT * 2); + /* * Verify that it handles interrupted threads properly. - * Use a large timeout and expect the thread to return quickly. + * Use a large timeout and expect the thread to return quickly + * upon interruption. */ - in = new SocketInputStream(source, 0); - Thread thread = new Thread(new ReadRunnable(in)); - thread.start(); - - try { - Thread.sleep(1000); - } catch (InterruptedException ignored) {} - + ((SocketInputStream)in).setTimeout(0); + TestingThread thread = new TestingThread(ctx) { + @Override + public void doWork() throws Exception { + try { + in.read(); + fail("Did not fail with interrupt"); + } catch (InterruptedIOException ste) { + LOG.info("Got expection while reading as expected : " + + ste.getMessage()); + } + } + }; + ctx.addThread(thread); + ctx.startThreads(); + // If the thread is interrupted before it calls read() + // then it throws ClosedByInterruptException due to + // some Java quirk. Waiting for it to call read() + // gets it into select(), so we get the expected + // InterruptedIOException. + Thread.sleep(1000); thread.interrupt(); + ctx.stop(); + + //make sure the channels are still open + assertTrue(source.isOpen()); + assertTrue(sink.isOpen()); + // Nevertheless, the output stream is closed, because + // a partial write may have succeeded (see comment in + // SocketOutputStream#write(byte[]), int, int) try { - thread.join(); - } catch (InterruptedException e) { - throw new IOException("Unexpected InterruptedException : " + e); + out.write(1); + fail("Did not throw"); + } catch (IOException ioe) { + GenericTestUtils.assertExceptionContains( + "stream is closed", ioe); } - //make sure the channels are still open - assertTrue(source.isOpen()); - assertTrue(sink.isOpen()); - out.close(); assertFalse(sink.isOpen()); Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestTableMapping.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestTableMapping.java?rev=1337645&r1=1337644&r2=1337645&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestTableMapping.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestTableMapping.java Sat May 12 20:52:34 2012 @@ -41,7 +41,7 @@ public class TestTableMapping { public void setUp() throws IOException { mappingFile = File.createTempFile(getClass().getSimpleName(), ".txt"); Files.write("a.b.c /rack1\n" + - "1.2.3\t/rack2\n", mappingFile, Charsets.UTF_8); + "1.2.3.4\t/rack2\n", mappingFile, Charsets.UTF_8); mappingFile.deleteOnExit(); } @@ -55,7 +55,7 @@ public class TestTableMapping { List<String> names = new ArrayList<String>(); names.add("a.b.c"); - names.add("1.2.3"); + names.add("1.2.3.4"); List<String> result = mapping.resolve(names); assertEquals(names.size(), result.size()); @@ -73,7 +73,7 @@ public class TestTableMapping { List<String> names = new ArrayList<String>(); names.add("a.b.c"); - names.add("1.2.3"); + names.add("1.2.3.4"); List<String> result1 = mapping.resolve(names); assertEquals(names.size(), result1.size()); @@ -96,7 +96,7 @@ public class TestTableMapping { List<String> names = new ArrayList<String>(); names.add("a.b.c"); - names.add("1.2.3"); + names.add("1.2.3.4"); List<String> result = mapping.resolve(names); assertEquals(names.size(), result.size()); @@ -114,7 +114,7 @@ public class TestTableMapping { List<String> names = new ArrayList<String>(); names.add("a.b.c"); - names.add("1.2.3"); + names.add("1.2.3.4"); List<String> result = mapping.resolve(names); assertEquals(names.size(), result.size()); @@ -134,7 +134,7 @@ public class TestTableMapping { List<String> names = new ArrayList<String>(); names.add("a.b.c"); - names.add("1.2.3"); + names.add("1.2.3.4"); List<String> result = mapping.resolve(names); assertEquals(names.size(), result.size()); Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestToken.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestToken.java?rev=1337645&r1=1337644&r2=1337645&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestToken.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestToken.java Sat May 12 20:52:34 2012 @@ -18,11 +18,15 @@ package org.apache.hadoop.security.token; +import static junit.framework.Assert.assertEquals; + import java.io.*; import java.util.Arrays; import org.apache.hadoop.io.*; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; +import org.apache.hadoop.security.token.delegation.TestDelegationToken.TestDelegationTokenIdentifier; +import org.apache.hadoop.security.token.delegation.TestDelegationToken.TestDelegationTokenSecretManager; import junit.framework.TestCase; @@ -94,5 +98,20 @@ public class TestToken extends TestCase checkUrlSafe(encode); } } + + public void testDecodeIdentifier() throws IOException { + TestDelegationTokenSecretManager secretManager = + new TestDelegationTokenSecretManager(0, 0, 0, 0); + secretManager.startThreads(); + TestDelegationTokenIdentifier id = new TestDelegationTokenIdentifier( + new Text("owner"), new Text("renewer"), new Text("realUser")); + + Token<TestDelegationTokenIdentifier> token = + new Token<TestDelegationTokenIdentifier>(id, secretManager); + TokenIdentifier idCopy = token.decodeIdentifier(); + + assertNotSame(id, idCopy); + assertEquals(id, idCopy); + } } Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestVersionUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestVersionUtil.java?rev=1337645&r1=1337644&r2=1337645&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestVersionUtil.java (original) +++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestVersionUtil.java Sat May 12 20:52:34 2012 @@ -19,7 +19,6 @@ package org.apache.hadoop.util; import static org.junit.Assert.*; -import org.apache.hadoop.test.GenericTestUtils; import org.junit.Test; public class TestVersionUtil { @@ -30,6 +29,8 @@ public class TestVersionUtil { assertEquals(0, VersionUtil.compareVersions("2.0.0", "2.0.0")); assertEquals(0, VersionUtil.compareVersions("2.0.0a", "2.0.0a")); assertEquals(0, VersionUtil.compareVersions("1", "1")); + assertEquals(0, VersionUtil.compareVersions( + "2.0.0-SNAPSHOT", "2.0.0-SNAPSHOT")); // Assert that lower versions are lower, and higher versions are higher. assertExpectedValues("1", "2.0.0"); @@ -52,6 +53,13 @@ public class TestVersionUtil { assertExpectedValues("1.0.0a2", "1.0.0a10"); assertExpectedValues("1.0", "1.a"); assertExpectedValues("1.0", "1.a0"); + + // Snapshot builds precede their eventual releases. + assertExpectedValues("1.0-SNAPSHOT", "1.0"); + assertExpectedValues("1.0", "1.0.0-SNAPSHOT"); + assertExpectedValues("1.0.0-SNAPSHOT", "1.0.0"); + assertExpectedValues("1.0.0", "1.0.1-SNAPSHOT"); + assertExpectedValues("1.0.1-SNAPSHOT", "1.0.1"); } private static void assertExpectedValues(String lower, String higher) {