Repository: hadoop Updated Branches: refs/heads/trunk 074259133 -> a0521bc83
HADOOP-11490. Expose truncate API via FileSystem and shell command. Contributed by Milan Desai. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a0521bc8 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a0521bc8 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a0521bc8 Branch: refs/heads/trunk Commit: a0521bc83a168a06f21314a9aff78630a576fc75 Parents: 0742591 Author: Konstantin V Shvachko <s...@apache.org> Authored: Wed Jan 21 15:58:58 2015 -0800 Committer: Konstantin V Shvachko <s...@apache.org> Committed: Wed Jan 21 16:00:49 2015 -0800 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 5 +- .../apache/hadoop/fs/ChecksumFileSystem.java | 5 + .../java/org/apache/hadoop/fs/FileSystem.java | 23 ++++ .../org/apache/hadoop/fs/FilterFileSystem.java | 5 + .../org/apache/hadoop/fs/HarFileSystem.java | 8 ++ .../apache/hadoop/fs/RawLocalFileSystem.java | 25 ++++ .../org/apache/hadoop/fs/shell/FsCommand.java | 1 + .../org/apache/hadoop/fs/shell/Truncate.java | 117 +++++++++++++++++++ .../apache/hadoop/fs/viewfs/ViewFileSystem.java | 13 +++ .../hadoop/hdfs/DistributedFileSystem.java | 9 +- .../hdfs/server/namenode/TestFileTruncate.java | 69 +++++++++++ 11 files changed, 271 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0521bc8/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index c54800f..66fd138 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -18,7 +18,10 @@ Trunk (Unreleased) HADOOP-6590. Add a username check for hadoop sub-commands (John Smith via aw) HADOOP-11353. Add support for .hadooprc (aw) - + + HADOOP-11490. Expose truncate API via FileSystem and shell command. + (Milan Desai via shv) + IMPROVEMENTS HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0521bc8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java index b6b865c..dddf0ce 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java @@ -352,6 +352,11 @@ public abstract class ChecksumFileSystem extends FilterFileSystem { throw new IOException("Not supported"); } + @Override + public boolean truncate(Path f, long newLength) throws IOException { + throw new IOException("Not supported"); + } + /** * Calculated the length of the checksum file in bytes. * @param size the length of the data file in bytes http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0521bc8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index 619f433..cfa5198 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -1317,6 +1317,29 @@ public abstract class FileSystem extends Configured implements Closeable { throw new IOException("rename from " + src + " to " + dst + " failed."); } } + + /** + * Truncate the file in the indicated path to the indicated size. + * <ul> + * <li>Fails if path is a directory. + * <li>Fails if path does not exist. + * <li>Fails if path is not closed. + * <li>Fails if new size is greater than current size. + * </ul> + * @param f The path to the file to be truncated + * @param newLength The size the file is to be truncated to + * + * @return <code>true</code> if the file has been truncated to the desired + * <code>newLength</code> and is immediately available to be reused for + * write operations such as <code>append</code>, or + * <code>false</code> if a background process of adjusting the length of + * the last block has been started, and clients should wait for it to + * complete before proceeding with further file updates. + */ + public boolean truncate(Path f, long newLength) throws IOException { + throw new UnsupportedOperationException("Not implemented by the " + + getClass().getSimpleName() + " FileSystem implementation"); + } /** * Delete a file http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0521bc8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java index 3d5a753..d4080ad 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java @@ -225,6 +225,11 @@ public class FilterFileSystem extends FileSystem { public boolean rename(Path src, Path dst) throws IOException { return fs.rename(src, dst); } + + @Override + public boolean truncate(Path f, final long newLength) throws IOException { + return fs.truncate(f, newLength); + } /** Delete a file */ @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0521bc8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java index 0fba268..e89bc49 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java @@ -761,6 +761,14 @@ public class HarFileSystem extends FileSystem { * Not implemented. */ @Override + public boolean truncate(Path f, long newLength) throws IOException { + throw new IOException("Har: truncate not allowed"); + } + + /** + * Not implemented. + */ + @Override public boolean delete(Path f, boolean recursive) throws IOException { throw new IOException("Har: delete not allowed"); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0521bc8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java index 75ef189..d7866b8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java @@ -371,6 +371,31 @@ public class RawLocalFileSystem extends FileSystem { } return FileUtil.copy(this, src, this, dst, true, getConf()); } + + @Override + public boolean truncate(Path f, final long newLength) throws IOException { + FileStatus status = getFileStatus(f); + if(status == null) { + throw new FileNotFoundException("File " + f + " not found"); + } + if(status.isDirectory()) { + throw new IOException("Cannot truncate a directory (=" + f + ")"); + } + long oldLength = status.getLen(); + if(newLength > oldLength) { + throw new IllegalArgumentException( + "Cannot truncate to a larger file size. Current size: " + oldLength + + ", truncate size: " + newLength + "."); + } + try (FileOutputStream out = new FileOutputStream(pathToFile(f), true)) { + try { + out.getChannel().truncate(newLength); + } catch(IOException e) { + throw new FSError(e); + } + } + return true; + } /** * Delete the given path to a file or directory. http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0521bc8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java index cc8fbb4..9515fde 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java @@ -60,6 +60,7 @@ abstract public class FsCommand extends Command { factory.registerCommands(Tail.class); factory.registerCommands(Test.class); factory.registerCommands(Touch.class); + factory.registerCommands(Truncate.class); factory.registerCommands(SnapshotCommands.class); factory.registerCommands(XAttrCommands.class); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0521bc8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Truncate.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Truncate.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Truncate.java new file mode 100644 index 0000000..9912863 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Truncate.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.shell; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.PathIsDirectoryException; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; + +/** + * Truncates a file to a new size + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class Truncate extends FsCommand { + public static void registerCommands(CommandFactory factory) { + factory.addClass(Truncate.class, "-truncate"); + } + + public static final String NAME = "truncate"; + public static final String USAGE = "[-w] <length> <path> ..."; + public static final String DESCRIPTION = + "Truncate all files that match the specified file pattern to the " + + "specified length.\n" + + "-w: Requests that the command wait for block recovery to complete, " + + "if necessary."; + + protected long newLength = -1; + protected List<PathData> waitList = new LinkedList<>(); + protected boolean waitOpt = false; + + @Override + protected void processOptions(LinkedList<String> args) throws IOException { + CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "w"); + cf.parse(args); + waitOpt = cf.getOpt("w"); + + try { + newLength = Long.parseLong(args.removeFirst()); + } catch(NumberFormatException nfe) { + displayWarning("Illegal length, a non-negative integer expected"); + throw nfe; + } + if(newLength < 0) { + throw new IllegalArgumentException("length must be >= 0"); + } + } + + @Override + protected void processArguments(LinkedList<PathData> args) + throws IOException { + super.processArguments(args); + if (waitOpt) waitForRecovery(); + } + + @Override + protected void processPath(PathData item) throws IOException { + if(item.stat.isDirectory()) { + throw new PathIsDirectoryException(item.toString()); + } + long oldLength = item.stat.getLen(); + if(newLength > oldLength) { + throw new IllegalArgumentException( + "Cannot truncate to a larger file size. Current size: " + oldLength + + ", truncate size: " + newLength + "."); + } + if(item.fs.truncate(item.path, newLength)) { + out.println("Truncated " + item + " to length: " + newLength); + } + else if(waitOpt) { + waitList.add(item); + } + else { + out.println("Truncating " + item + " to length: " + newLength + ". " + + "Wait for block recovery to complete before further updating this " + + "file."); + } + } + + /** + * Wait for all files in waitList to have length equal to newLength. + */ + private void waitForRecovery() throws IOException { + for(PathData item : waitList) { + out.println("Waiting for " + item + " ..."); + out.flush(); + + for(;;) { + item.refreshStatus(); + if(item.stat.getLen() == newLength) break; + try {Thread.sleep(1000);} catch(InterruptedException ignored) {} + } + + out.println("Truncated " + item + " to length: " + newLength); + out.flush(); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0521bc8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 963289f..0f77f47 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -446,6 +446,14 @@ public class ViewFileSystem extends FileSystem { return resSrc.targetFileSystem.rename(resSrc.remainingPath, resDst.remainingPath); } + + @Override + public boolean truncate(final Path f, final long newLength) + throws IOException { + InodeTree.ResolveResult<FileSystem> res = + fsState.resolve(getUriPath(f), true); + return res.targetFileSystem.truncate(f, newLength); + } @Override public void setOwner(final Path f, final String username, @@ -834,6 +842,11 @@ public class ViewFileSystem extends FileSystem { } @Override + public boolean truncate(Path f, long newLength) throws IOException { + throw readOnlyMountTable("truncate", f); + } + + @Override public void setOwner(Path f, String username, String groupname) throws AccessControlException, IOException { checkPathIsSlash(f); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0521bc8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 6284f61..654e2f9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -627,14 +627,7 @@ public class DistributedFileSystem extends FileSystem { } } - /** - * Truncate the file in the indicated path to the indicated size. - * @param f The path to the file to be truncated - * @param newLength The size the file is to be truncated to - * - * @return true if and client does not need to wait for block recovery, - * false if client needs to wait for block recovery. - */ + @Override public boolean truncate(Path f, final long newLength) throws IOException { statistics.incrementWriteOps(1); return dfs.truncate(getPathName(f), newLength); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0521bc8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java index 1f854d1..5498b12 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java @@ -38,6 +38,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.AppendTestUtil; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -55,6 +56,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Time; +import org.apache.hadoop.util.ToolRunner; import org.apache.log4j.Level; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -676,6 +678,73 @@ public class TestFileTruncate { fs.delete(parent, true); } + @Test + public void testTruncateShellCommand() throws Exception { + final Path parent = new Path("/test"); + final Path src = new Path("/test/testTruncateShellCommand"); + final int oldLength = 2*BLOCK_SIZE + 1; + final int newLength = BLOCK_SIZE + 1; + + String[] argv = + new String[]{"-truncate", String.valueOf(newLength), src.toString()}; + runTruncateShellCommand(src, oldLength, argv); + + // wait for block recovery + checkBlockRecovery(src); + assertThat(fs.getFileStatus(src).getLen(), is((long) newLength)); + fs.delete(parent, true); + } + + @Test + public void testTruncateShellCommandOnBlockBoundary() throws Exception { + final Path parent = new Path("/test"); + final Path src = new Path("/test/testTruncateShellCommandOnBoundary"); + final int oldLength = 2 * BLOCK_SIZE; + final int newLength = BLOCK_SIZE; + + String[] argv = + new String[]{"-truncate", String.valueOf(newLength), src.toString()}; + runTruncateShellCommand(src, oldLength, argv); + + // shouldn't need to wait for block recovery + assertThat(fs.getFileStatus(src).getLen(), is((long) newLength)); + fs.delete(parent, true); + } + + @Test + public void testTruncateShellCommandWithWaitOption() throws Exception { + final Path parent = new Path("/test"); + final Path src = new Path("/test/testTruncateShellCommandWithWaitOption"); + final int oldLength = 2 * BLOCK_SIZE + 1; + final int newLength = BLOCK_SIZE + 1; + + String[] argv = new String[]{"-truncate", "-w", String.valueOf(newLength), + src.toString()}; + runTruncateShellCommand(src, oldLength, argv); + + // shouldn't need to wait for block recovery + assertThat(fs.getFileStatus(src).getLen(), is((long) newLength)); + fs.delete(parent, true); + } + + private void runTruncateShellCommand(Path src, int oldLength, + String[] shellOpts) throws Exception { + // create file and write data + writeContents(AppendTestUtil.initBuffer(oldLength), oldLength, src); + assertThat(fs.getFileStatus(src).getLen(), is((long)oldLength)); + + // truncate file using shell + FsShell shell = null; + try { + shell = new FsShell(conf); + assertThat(ToolRunner.run(shell, shellOpts), is(0)); + } finally { + if(shell != null) { + shell.close(); + } + } + } + static void writeContents(byte[] contents, int fileLength, Path p) throws IOException { FSDataOutputStream out = fs.create(p, true, BLOCK_SIZE, REPLICATION,