Author: acmurthy Date: Wed Nov 28 15:17:21 2012 New Revision: 1414750 URL: http://svn.apache.org/viewvc?rev=1414750&view=rev Log: Rebased on trunk.
Added: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathAccessDeniedException.java - copied unchanged from r1414746, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathAccessDeniedException.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathExistsException.java - copied unchanged from r1414746, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathExistsException.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIOException.java - copied unchanged from r1414746, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIOException.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIsDirectoryException.java - copied unchanged from r1414746, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIsDirectoryException.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIsNotDirectoryException.java - copied unchanged from r1414746, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIsNotDirectoryException.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIsNotEmptyDirectoryException.java - copied unchanged from r1414746, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIsNotEmptyDirectoryException.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathNotFoundException.java - copied unchanged from r1414746, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathNotFoundException.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathOperationException.java - copied unchanged from r1414746, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathOperationException.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathPermissionException.java - copied unchanged from r1414746, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathPermissionException.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDelegationTokenRenewer.java - copied unchanged from r1414746, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDelegationTokenRenewer.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java - copied unchanged from r1414746, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java Removed: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathExceptions.java Modified: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed) hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/docs/ (props changed) hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/ (props changed) hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Mkdir.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Test.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touchz.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/core/ (props changed) hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathExceptions.java Modified: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1414750&r1=1414749&r2=1414750&view=diff ============================================================================== --- hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/CHANGES.txt Wed Nov 28 15:17:21 2012 @@ -132,7 +132,14 @@ Trunk (Unreleased) HADOOP-9004. Allow security unit tests to use external KDC. (Stephen Chu via suresh) - HADOOP-6616. Improve documentation for rack awareness. (Adam Faris via jghoman) + HADOOP-6616. Improve documentation for rack awareness. (Adam Faris via + jghoman) + + HADOOP-9075. FileContext#FSLinkResolver should be made static. + (Arpit Agarwal via suresh) + + HADOOP-9093. Move all the Exception in PathExceptions to o.a.h.fs package. + (suresh) BUG FIXES @@ -373,6 +380,9 @@ Release 2.0.3-alpha - Unreleased HADOOP-9035. Generalize setup of LoginContext (daryn via bobby) + HADOOP-9042. Add a test for umask in FileSystemContractBaseTest. + (Colin McCabe via eli) + OPTIMIZATIONS HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang @@ -437,6 +447,12 @@ Release 2.0.3-alpha - Unreleased HADOOP-6607. Add different variants of non caching HTTP headers. (tucu) + HADOOP-9049. DelegationTokenRenewer needs to be Singleton and FileSystems + should register/deregister to/from. (Karthik Kambatla via tomwhite) + + HADOOP-9064. Augment DelegationTokenRenewer API to cancel the tokens on + calls to removeRenewAction. (kkambatl via tucu) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES @@ -1139,6 +1155,15 @@ Release 0.23.6 - UNRELEASED BUG FIXES + HADOOP-9072. Hadoop-Common-0.23-Build Fails to build in Jenkins + (Robert Parker via tgraves) + + HADOOP-8992. Enhance unit-test coverage of class HarFileSystem (Ivan A. + Veselovsky via bobby) + + HADOOP-9038. unit-tests for AllocatorPerContext.PathIterator (Ivan A. + Veselovsky via bobby) + Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES Propchange: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/CHANGES.txt ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1411704-1414746 Modified: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml?rev=1414750&r1=1414749&r2=1414750&view=diff ============================================================================== --- hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml (original) +++ hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml Wed Nov 28 15:17:21 2012 @@ -291,5 +291,13 @@ <Field name="previousSnapshot" /> <Bug pattern="IS2_INCONSISTENT_SYNC" /> </Match> - + <!-- + The method uses a generic type T that extends two other types + T1 and T2. Findbugs complains of a cast from T1 to T2. + --> + <Match> + <Class name="org.apache.hadoop.fs.DelegationTokenRenewer" /> + <Method name="removeRenewAction" /> + <Bug pattern="BC_UNCONFIRMED_CAST" /> + </Match> </FindBugsFilter> Propchange: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/docs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1411704-1414746 Propchange: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1411704-1414746 Modified: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java?rev=1414750&r1=1414749&r2=1414750&view=diff ============================================================================== --- hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java (original) +++ hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java Wed Nov 28 15:17:21 2012 @@ -24,6 +24,8 @@ import java.util.concurrent.DelayQueue; import java.util.concurrent.Delayed; import java.util.concurrent.TimeUnit; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; @@ -33,8 +35,11 @@ import org.apache.hadoop.util.Time; * A daemon thread that waits for the next file system to renew. */ @InterfaceAudience.Private -public class DelegationTokenRenewer<T extends FileSystem & DelegationTokenRenewer.Renewable> +public class DelegationTokenRenewer extends Thread { + private static final Log LOG = LogFactory + .getLog(DelegationTokenRenewer.class); + /** The renewable interface used by the renewer. */ public interface Renewable { /** @return the renew token. */ @@ -93,7 +98,7 @@ public class DelegationTokenRenewer<T ex * @param newTime the new time */ private void updateRenewalTime() { - renewalTime = RENEW_CYCLE + Time.now(); + renewalTime = renewCycle + Time.now(); } /** @@ -134,34 +139,82 @@ public class DelegationTokenRenewer<T ex } /** Wait for 95% of a day between renewals */ - private static final int RENEW_CYCLE = 24 * 60 * 60 * 950; + private static final int RENEW_CYCLE = 24 * 60 * 60 * 950; - private DelayQueue<RenewAction<T>> queue = new DelayQueue<RenewAction<T>>(); + @InterfaceAudience.Private + protected static int renewCycle = RENEW_CYCLE; - public DelegationTokenRenewer(final Class<T> clazz) { + /** Queue to maintain the RenewActions to be processed by the {@link #run()} */ + private volatile DelayQueue<RenewAction<?>> queue = new DelayQueue<RenewAction<?>>(); + + /** + * Create the singleton instance. However, the thread can be started lazily in + * {@link #addRenewAction(FileSystem)} + */ + private static DelegationTokenRenewer INSTANCE = null; + + private DelegationTokenRenewer(final Class<? extends FileSystem> clazz) { super(clazz.getSimpleName() + "-" + DelegationTokenRenewer.class.getSimpleName()); setDaemon(true); } + public static synchronized DelegationTokenRenewer getInstance() { + if (INSTANCE == null) { + INSTANCE = new DelegationTokenRenewer(FileSystem.class); + } + return INSTANCE; + } + /** Add a renew action to the queue. */ - public void addRenewAction(final T fs) { + public synchronized <T extends FileSystem & Renewable> void addRenewAction(final T fs) { queue.add(new RenewAction<T>(fs)); + if (!isAlive()) { + start(); + } + } + + /** + * Remove the associated renew action from the queue + * + * @throws IOException + */ + public synchronized <T extends FileSystem & Renewable> void removeRenewAction( + final T fs) throws IOException { + for (RenewAction<?> action : queue) { + if (action.weakFs.get() == fs) { + try { + fs.getRenewToken().cancel(fs.getConf()); + } catch (InterruptedException ie) { + LOG.error("Interrupted while canceling token for " + fs.getUri() + + "filesystem"); + if (LOG.isDebugEnabled()) { + LOG.debug(ie.getStackTrace()); + } + } + queue.remove(action); + return; + } + } } + @SuppressWarnings("static-access") @Override public void run() { for(;;) { - RenewAction<T> action = null; + RenewAction<?> action = null; try { - action = queue.take(); - if (action.renew()) { - action.updateRenewalTime(); - queue.add(action); + synchronized (this) { + action = queue.take(); + if (action.renew()) { + action.updateRenewalTime(); + queue.add(action); + } } } catch (InterruptedException ie) { return; } catch (Exception ie) { - T.LOG.warn("Failed to renew token, action=" + action, ie); + action.weakFs.get().LOG.warn("Failed to renew token, action=" + action, + ie); } } } Modified: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java?rev=1414750&r1=1414749&r2=1414750&view=diff ============================================================================== --- hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java (original) +++ hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java Wed Nov 28 15:17:21 2012 @@ -1119,7 +1119,7 @@ public final class FileContext { * @param target The symlink's absolute target * @return Fully qualified version of the target. */ - private Path qualifySymlinkTarget(final AbstractFileSystem pathFS, + private static Path qualifySymlinkTarget(final AbstractFileSystem pathFS, Path pathWithLink, Path target) { // NB: makeQualified uses the target's scheme and authority, if // specified, and the scheme and authority of pathFS, if not. @@ -2321,7 +2321,7 @@ public final class FileContext { * Class used to perform an operation on and resolve symlinks in a * path. The operation may potentially span multiple file systems. */ - protected abstract class FSLinkResolver<T> { + protected static abstract class FSLinkResolver<T> { // The maximum number of symbolic link components in a path private static final int MAX_PATH_LINKS = 32; Modified: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java?rev=1414750&r1=1414749&r2=1414750&view=diff ============================================================================== --- hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java (original) +++ hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java Wed Nov 28 15:17:21 2012 @@ -584,13 +584,6 @@ public class HarFileSystem extends Filte public String getName() { return name; } - - public List<String> getChildren() { - return children; - } - public String getFileName() { - return name; - } public String getPartName() { return partName; } @@ -662,15 +655,6 @@ public class HarFileSystem extends Filte hstatus.getStartIndex(), hstatus.getLength(), bufferSize); } - /* - * create throws an exception in Har filesystem. - * The archive once created cannot be changed. - */ - public FSDataOutputStream create(Path f, int bufferSize) - throws IOException { - throw new IOException("Har: Create not allowed"); - } - @Override public FSDataOutputStream create(Path f, FsPermission permission, @@ -1106,4 +1090,11 @@ public class HarFileSystem extends Filte } } } + + /* + * testing purposes only: + */ + HarMetaData getMetadata() { + return metadata; + } } Modified: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java?rev=1414750&r1=1414749&r2=1414750&view=diff ============================================================================== --- hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java (original) +++ hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java Wed Nov 28 15:17:21 2012 @@ -481,12 +481,15 @@ public class LocalDirAllocator { @Override public Path next() { - Path result = next; + final Path result = next; try { advance(); } catch (IOException ie) { throw new RuntimeException("Can't check existance of " + next, ie); } + if (result == null) { + throw new NoSuchElementException(); + } return result; } Modified: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java?rev=1414750&r1=1414749&r2=1414750&view=diff ============================================================================== --- hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java (original) +++ hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java Wed Nov 28 15:17:21 2012 @@ -34,7 +34,7 @@ import org.apache.hadoop.classification. import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.shell.PathExceptions.PathNotFoundException; +import org.apache.hadoop.fs.PathNotFoundException; import org.apache.hadoop.util.StringUtils; /** Modified: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java?rev=1414750&r1=1414749&r2=1414750&view=diff ============================================================================== --- hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java (original) +++ hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java Wed Nov 28 15:17:21 2012 @@ -27,12 +27,12 @@ import org.apache.hadoop.fs.FSDataOutput import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FilterFileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.shell.PathExceptions.PathExistsException; -import org.apache.hadoop.fs.shell.PathExceptions.PathIOException; -import org.apache.hadoop.fs.shell.PathExceptions.PathIsDirectoryException; -import org.apache.hadoop.fs.shell.PathExceptions.PathIsNotDirectoryException; -import org.apache.hadoop.fs.shell.PathExceptions.PathNotFoundException; -import org.apache.hadoop.fs.shell.PathExceptions.PathOperationException; +import org.apache.hadoop.fs.PathExistsException; +import org.apache.hadoop.fs.PathIOException; +import org.apache.hadoop.fs.PathIsDirectoryException; +import org.apache.hadoop.fs.PathIsNotDirectoryException; +import org.apache.hadoop.fs.PathNotFoundException; +import org.apache.hadoop.fs.PathOperationException; import org.apache.hadoop.io.IOUtils; /** Modified: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java?rev=1414750&r1=1414749&r2=1414750&view=diff ============================================================================== --- hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java (original) +++ hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java Wed Nov 28 15:17:21 2012 @@ -27,7 +27,7 @@ import org.apache.hadoop.classification. import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.shell.PathExceptions.PathIsDirectoryException; +import org.apache.hadoop.fs.PathIsDirectoryException; import org.apache.hadoop.io.IOUtils; /** Various commands for copy files */ Modified: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java?rev=1414750&r1=1414749&r2=1414750&view=diff ============================================================================== --- hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java (original) +++ hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java Wed Nov 28 15:17:21 2012 @@ -24,11 +24,11 @@ import java.util.LinkedList; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.PathIOException; +import org.apache.hadoop.fs.PathIsDirectoryException; +import org.apache.hadoop.fs.PathIsNotDirectoryException; +import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException; import org.apache.hadoop.fs.Trash; -import org.apache.hadoop.fs.shell.PathExceptions.PathIOException; -import org.apache.hadoop.fs.shell.PathExceptions.PathIsDirectoryException; -import org.apache.hadoop.fs.shell.PathExceptions.PathIsNotDirectoryException; -import org.apache.hadoop.fs.shell.PathExceptions.PathIsNotEmptyDirectoryException; /** * Classes that delete paths Modified: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java?rev=1414750&r1=1414749&r2=1414750&view=diff ============================================================================== --- hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java (original) +++ hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java Wed Nov 28 15:17:21 2012 @@ -38,7 +38,7 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.shell.PathExceptions.PathIsDirectoryException; +import org.apache.hadoop.fs.PathIsDirectoryException; import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.IOUtils; Modified: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Mkdir.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Mkdir.java?rev=1414750&r1=1414749&r2=1414750&view=diff ============================================================================== --- hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Mkdir.java (original) +++ hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Mkdir.java Wed Nov 28 15:17:21 2012 @@ -24,10 +24,10 @@ import java.util.LinkedList; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.shell.PathExceptions.PathExistsException; -import org.apache.hadoop.fs.shell.PathExceptions.PathIOException; -import org.apache.hadoop.fs.shell.PathExceptions.PathIsNotDirectoryException; -import org.apache.hadoop.fs.shell.PathExceptions.PathNotFoundException; +import org.apache.hadoop.fs.PathExistsException; +import org.apache.hadoop.fs.PathIOException; +import org.apache.hadoop.fs.PathIsNotDirectoryException; +import org.apache.hadoop.fs.PathNotFoundException; /** * Create the given dir Modified: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java?rev=1414750&r1=1414749&r2=1414750&view=diff ============================================================================== --- hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java (original) +++ hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java Wed Nov 28 15:17:21 2012 @@ -23,8 +23,8 @@ import java.util.LinkedList; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.PathIOException; import org.apache.hadoop.fs.shell.CopyCommands.CopyFromLocal; -import org.apache.hadoop.fs.shell.PathExceptions.PathIOException; /** Various commands for moving files */ @InterfaceAudience.Private Modified: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java?rev=1414750&r1=1414749&r2=1414750&view=diff ============================================================================== --- hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java (original) +++ hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java Wed Nov 28 15:17:21 2012 @@ -32,10 +32,10 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.shell.PathExceptions.PathIOException; -import org.apache.hadoop.fs.shell.PathExceptions.PathIsDirectoryException; -import org.apache.hadoop.fs.shell.PathExceptions.PathIsNotDirectoryException; -import org.apache.hadoop.fs.shell.PathExceptions.PathNotFoundException; +import org.apache.hadoop.fs.PathIOException; +import org.apache.hadoop.fs.PathIsDirectoryException; +import org.apache.hadoop.fs.PathIsNotDirectoryException; +import org.apache.hadoop.fs.PathNotFoundException; /** * Encapsulates a Path (path), its FileStatus (stat), and its FileSystem (fs). Modified: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java?rev=1414750&r1=1414749&r2=1414750&view=diff ============================================================================== --- hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java (original) +++ hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java Wed Nov 28 15:17:21 2012 @@ -25,7 +25,7 @@ import java.util.List; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.BlockLocation; -import org.apache.hadoop.fs.shell.PathExceptions.PathIOException; +import org.apache.hadoop.fs.PathIOException; /** * Modifies the replication factor Modified: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java?rev=1414750&r1=1414749&r2=1414750&view=diff ============================================================================== --- hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java (original) +++ hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java Wed Nov 28 15:17:21 2012 @@ -25,7 +25,7 @@ import java.util.List; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.shell.PathExceptions.PathIsDirectoryException; +import org.apache.hadoop.fs.PathIsDirectoryException; import org.apache.hadoop.io.IOUtils; /** Modified: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Test.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Test.java?rev=1414750&r1=1414749&r2=1414750&view=diff ============================================================================== --- hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Test.java (original) +++ hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Test.java Wed Nov 28 15:17:21 2012 @@ -23,7 +23,6 @@ import java.util.LinkedList; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.fs.shell.PathExceptions.PathNotFoundException; /** * Perform shell-like file tests Modified: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touchz.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touchz.java?rev=1414750&r1=1414749&r2=1414750&view=diff ============================================================================== --- hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touchz.java (original) +++ hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touchz.java Wed Nov 28 15:17:21 2012 @@ -23,9 +23,9 @@ import java.util.LinkedList; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.fs.shell.PathExceptions.PathIOException; -import org.apache.hadoop.fs.shell.PathExceptions.PathIsDirectoryException; -import org.apache.hadoop.fs.shell.PathExceptions.PathNotFoundException; +import org.apache.hadoop.fs.PathIOException; +import org.apache.hadoop.fs.PathIsDirectoryException; +import org.apache.hadoop.fs.PathNotFoundException; /** * Unix touch like commands Propchange: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/core/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1411704-1414746 Modified: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java?rev=1414750&r1=1414749&r2=1414750&view=diff ============================================================================== --- hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java (original) +++ hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java Wed Nov 28 15:17:21 2012 @@ -23,11 +23,13 @@ import java.io.IOException; import junit.framework.TestCase; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; /** * <p> @@ -43,7 +45,7 @@ import org.apache.hadoop.fs.Path; * </p> */ public abstract class FileSystemContractBaseTest extends TestCase { - + protected final static String TEST_UMASK = "062"; protected FileSystem fs; protected byte[] data = new byte[getBlockSize() * 2]; // two blocks of data { @@ -151,7 +153,26 @@ public abstract class FileSystemContract assertFalse(fs.exists(testDeepSubDir)); } - + + public void testMkdirsWithUmask() throws Exception { + if (fs.getScheme().equals("s3") || fs.getScheme().equals("s3n")) { + // skip permission tests for S3FileSystem until HDFS-1333 is fixed. + return; + } + Configuration conf = fs.getConf(); + String oldUmask = conf.get(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY); + try { + conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, TEST_UMASK); + final Path dir = new Path("/test/newDir"); + assertTrue(fs.mkdirs(dir, new FsPermission((short)0777))); + FileStatus status = fs.getFileStatus(dir); + assertTrue(status.isDirectory()); + assertEquals((short)0715, status.getPermission().toShort()); + } finally { + conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, oldUmask); + } + } + public void testGetFileStatusThrowsExceptionForNonExistentFile() throws Exception { try { Modified: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java?rev=1414750&r1=1414749&r2=1414750&view=diff ============================================================================== --- hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java (original) +++ hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java Wed Nov 28 15:17:21 2012 @@ -22,6 +22,8 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.Arrays; import java.util.Collection; +import java.util.Iterator; +import java.util.NoSuchElementException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.Shell; @@ -32,6 +34,7 @@ import org.junit.runners.Parameterized.P import org.junit.Test; import static org.junit.Assert.*; +import static org.junit.Assume.*; /** This test LocalDirAllocator works correctly; * Every test case uses different buffer dirs to @@ -318,7 +321,7 @@ public class TestLocalDirAllocator { */ @Test public void testNoSideEffects() throws IOException { - if (isWindows) return; + assumeTrue(!isWindows); String dir = buildBufferDir(ROOT, 0); try { conf.set(CONTEXT, dir); @@ -339,8 +342,7 @@ public class TestLocalDirAllocator { */ @Test public void testGetLocalPathToRead() throws IOException { - if (isWindows) - return; + assumeTrue(!isWindows); String dir = buildBufferDir(ROOT, 0); try { conf.set(CONTEXT, dir); @@ -354,7 +356,60 @@ public class TestLocalDirAllocator { Shell.execCommand(new String[] { "chmod", "u+w", BUFFER_DIR_ROOT }); rmBufferDirs(); } + } + + /** + * Test that {@link LocalDirAllocator#getAllLocalPathsToRead(String, Configuration)} + * returns correct filenames and "file" schema. + * + * @throws IOException + */ + @Test + public void testGetAllLocalPathsToRead() throws IOException { + assumeTrue(!isWindows); + + String dir0 = buildBufferDir(ROOT, 0); + String dir1 = buildBufferDir(ROOT, 1); + try { + conf.set(CONTEXT, dir0 + "," + dir1); + assertTrue(localFs.mkdirs(new Path(dir0))); + assertTrue(localFs.mkdirs(new Path(dir1))); + + localFs.create(new Path(dir0 + Path.SEPARATOR + FILENAME)); + localFs.create(new Path(dir1 + Path.SEPARATOR + FILENAME)); + + // check both the paths are returned as paths to read: + final Iterable<Path> pathIterable = dirAllocator.getAllLocalPathsToRead(FILENAME, conf); + int count = 0; + for (final Path p: pathIterable) { + count++; + assertEquals(FILENAME, p.getName()); + assertEquals("file", p.getFileSystem(conf).getUri().getScheme()); + } + assertEquals(2, count); + // test #next() while no element to iterate any more: + try { + Path p = pathIterable.iterator().next(); + assertFalse("NoSuchElementException must be thrown, but returned ["+p + +"] instead.", true); // exception expected + } catch (NoSuchElementException nsee) { + // okay + } + + // test modification not allowed: + final Iterable<Path> pathIterable2 = dirAllocator.getAllLocalPathsToRead(FILENAME, conf); + final Iterator<Path> it = pathIterable2.iterator(); + try { + it.remove(); + assertFalse(true); // exception expected + } catch (UnsupportedOperationException uoe) { + // okay + } + } finally { + Shell.execCommand(new String[] { "chmod", "u+w", BUFFER_DIR_ROOT }); + rmBufferDirs(); + } } @Test Modified: hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathExceptions.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathExceptions.java?rev=1414750&r1=1414749&r2=1414750&view=diff ============================================================================== --- hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathExceptions.java (original) +++ hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathExceptions.java Wed Nov 28 15:17:21 2012 @@ -23,7 +23,7 @@ import static org.junit.Assert.assertEqu import java.io.IOException; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.shell.PathExceptions.PathIOException; +import org.apache.hadoop.fs.PathIOException; import org.junit.Test; public class TestPathExceptions {