HADOOP-9214. Create a new touch command to allow modifying atime and mtime. Contributed by Hrishikesh Gadre.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/60ffec9f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/60ffec9f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/60ffec9f Branch: refs/heads/HDFS-12943 Commit: 60ffec9f7921a50aff20434c1042b16fa59240f7 Parents: a17eed1 Author: Xiao Chen <x...@apache.org> Authored: Fri Aug 17 10:53:22 2018 -0700 Committer: Xiao Chen <x...@apache.org> Committed: Fri Aug 17 11:18:09 2018 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/fs/shell/FsCommand.java | 2 +- .../java/org/apache/hadoop/fs/shell/Touch.java | 85 -------- .../apache/hadoop/fs/shell/TouchCommands.java | 198 +++++++++++++++++++ .../src/site/markdown/FileSystemShell.md | 32 +++ .../org/apache/hadoop/fs/TestFsShellTouch.java | 103 ++++++++++ .../src/test/resources/testConf.xml | 51 +++++ 6 files changed, 385 insertions(+), 86 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/60ffec9f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java index 4a13414..784bbf3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java @@ -66,7 +66,7 @@ abstract public class FsCommand extends Command { factory.registerCommands(Tail.class); factory.registerCommands(Head.class); factory.registerCommands(Test.class); - factory.registerCommands(Touch.class); + factory.registerCommands(TouchCommands.class); factory.registerCommands(Truncate.class); factory.registerCommands(SnapshotCommands.class); factory.registerCommands(XAttrCommands.class); http://git-wip-us.apache.org/repos/asf/hadoop/blob/60ffec9f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touch.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touch.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touch.java deleted file mode 100644 index a6c751e..0000000 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touch.java +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.shell; - -import java.io.IOException; -import java.util.LinkedList; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.fs.PathIOException; -import org.apache.hadoop.fs.PathIsDirectoryException; -import org.apache.hadoop.fs.PathNotFoundException; - -/** - * Unix touch like commands - */ -@InterfaceAudience.Private -@InterfaceStability.Unstable - -class Touch extends FsCommand { - public static void registerCommands(CommandFactory factory) { - factory.addClass(Touchz.class, "-touchz"); - } - - /** - * (Re)create zero-length file at the specified path. - * This will be replaced by a more UNIX-like touch when files may be - * modified. - */ - public static class Touchz extends Touch { - public static final String NAME = "touchz"; - public static final String USAGE = "<path> ..."; - public static final String DESCRIPTION = - "Creates a file of zero length " + - "at <path> with current time as the timestamp of that <path>. " + - "An error is returned if the file exists with non-zero length\n"; - - @Override - protected void processOptions(LinkedList<String> args) { - CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE); - cf.parse(args); - } - - @Override - protected void processPath(PathData item) throws IOException { - if (item.stat.isDirectory()) { - // TODO: handle this - throw new PathIsDirectoryException(item.toString()); - } - if (item.stat.getLen() != 0) { - throw new PathIOException(item.toString(), "Not a zero-length file"); - } - touchz(item); - } - - @Override - protected void processNonexistentPath(PathData item) throws IOException { - if (!item.parentExists()) { - throw new PathNotFoundException(item.toString()) - .withFullyQualifiedPath(item.path.toUri().toString()); - } - touchz(item); - } - - private void touchz(PathData item) throws IOException { - item.fs.create(item.path).close(); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/60ffec9f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/TouchCommands.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/TouchCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/TouchCommands.java new file mode 100644 index 0000000..be174b5 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/TouchCommands.java @@ -0,0 +1,198 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.shell; + +import java.io.IOException; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.LinkedList; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.PathIOException; +import org.apache.hadoop.fs.PathIsDirectoryException; +import org.apache.hadoop.fs.PathNotFoundException; +import org.apache.hadoop.util.StringUtils; + +import com.google.common.annotations.VisibleForTesting; + +/** + * Unix touch like commands + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable + +public class TouchCommands extends FsCommand { + public static void registerCommands(CommandFactory factory) { + factory.addClass(Touchz.class, "-touchz"); + factory.addClass(Touch.class, "-touch"); + } + + /** + * (Re)create zero-length file at the specified path. + * This will be replaced by a more UNIX-like touch when files may be + * modified. + */ + public static class Touchz extends TouchCommands { + public static final String NAME = "touchz"; + public static final String USAGE = "<path> ..."; + public static final String DESCRIPTION = + "Creates a file of zero length " + + "at <path> with current time as the timestamp of that <path>. " + + "An error is returned if the file exists with non-zero length\n"; + + @Override + protected void processOptions(LinkedList<String> args) { + CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE); + cf.parse(args); + } + + @Override + protected void processPath(PathData item) throws IOException { + if (item.stat.isDirectory()) { + // TODO: handle this + throw new PathIsDirectoryException(item.toString()); + } + if (item.stat.getLen() != 0) { + throw new PathIOException(item.toString(), "Not a zero-length file"); + } + touchz(item); + } + + @Override + protected void processNonexistentPath(PathData item) throws IOException { + if (!item.parentExists()) { + throw new PathNotFoundException(item.toString()) + .withFullyQualifiedPath(item.path.toUri().toString()); + } + touchz(item); + } + + private void touchz(PathData item) throws IOException { + item.fs.create(item.path).close(); + } + } + + /** + * A UNIX like touch command. + */ + public static class Touch extends TouchCommands { + private static final String OPTION_CHANGE_ONLY_MODIFICATION_TIME = "m"; + private static final String OPTION_CHANGE_ONLY_ACCESS_TIME = "a"; + private static final String OPTION_USE_TIMESTAMP = "t"; + private static final String OPTION_DO_NOT_CREATE_FILE = "c"; + + public static final String NAME = "touch"; + public static final String USAGE = "[-" + OPTION_CHANGE_ONLY_ACCESS_TIME + + "] [-" + OPTION_CHANGE_ONLY_MODIFICATION_TIME + "] [-" + + OPTION_USE_TIMESTAMP + " TIMESTAMP ] [-" + OPTION_DO_NOT_CREATE_FILE + + "] <path> ..."; + public static final String DESCRIPTION = + "Updates the access and modification times of the file specified by the" + + " <path> to the current time. If the file does not exist, then a zero" + + " length file is created at <path> with current time as the timestamp" + + " of that <path>.\n" + + "-" + OPTION_CHANGE_ONLY_ACCESS_TIME + + " Change only the access time \n" + "-" + + OPTION_CHANGE_ONLY_MODIFICATION_TIME + + " Change only the modification time \n" + "-" + + OPTION_USE_TIMESTAMP + " TIMESTAMP" + + " Use specified timestamp (in format yyyyMMddHHmmss) instead of current time \n" + + "-" + OPTION_DO_NOT_CREATE_FILE + " Do not create any files"; + + private boolean changeModTime = false; + private boolean changeAccessTime = false; + private boolean doNotCreate = false; + private String timestamp; + private final SimpleDateFormat dateFormat = + new SimpleDateFormat("yyyyMMdd:HHmmss"); + + @InterfaceAudience.Private + @VisibleForTesting + public DateFormat getDateFormat() { + return dateFormat; + } + + @Override + protected void processOptions(LinkedList<String> args) { + this.timestamp = + StringUtils.popOptionWithArgument("-" + OPTION_USE_TIMESTAMP, args); + + CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, + OPTION_USE_TIMESTAMP, OPTION_CHANGE_ONLY_ACCESS_TIME, + OPTION_CHANGE_ONLY_MODIFICATION_TIME); + cf.parse(args); + this.changeModTime = cf.getOpt(OPTION_CHANGE_ONLY_MODIFICATION_TIME); + this.changeAccessTime = cf.getOpt(OPTION_CHANGE_ONLY_ACCESS_TIME); + this.doNotCreate = cf.getOpt(OPTION_DO_NOT_CREATE_FILE); + } + + @Override + protected void processPath(PathData item) throws IOException { + if (item.stat.isDirectory()) { + throw new PathIsDirectoryException(item.toString()); + } + touch(item); + } + + @Override + protected void processNonexistentPath(PathData item) throws IOException { + if (!item.parentExists()) { + throw new PathNotFoundException(item.toString()) + .withFullyQualifiedPath(item.path.toUri().toString()); + } + touch(item); + } + + private void touch(PathData item) throws IOException { + if (!item.fs.exists(item.path)) { + if (doNotCreate) { + return; + } + item.fs.create(item.path).close(); + if (timestamp != null) { + // update the time only if user specified a timestamp using -t option. + updateTime(item); + } + } else { + updateTime(item); + } + } + + private void updateTime(PathData item) throws IOException { + long time = System.currentTimeMillis(); + if (timestamp != null) { + try { + time = dateFormat.parse(timestamp).getTime(); + } catch (ParseException e) { + throw new IllegalArgumentException( + "Unable to parse the specified timestamp " + timestamp, e); + } + } + if (changeModTime ^ changeAccessTime) { + long atime = changeModTime ? -1 : time; + long mtime = changeAccessTime ? -1 : time; + item.fs.setTimes(item.path, mtime, atime); + } else { + item.fs.setTimes(item.path, time, time); + } + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/60ffec9f/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md index ec9d3c3..d9567b9 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md @@ -741,6 +741,38 @@ Usage: `hadoop fs -text <src> ` Takes a source file and outputs the file in text format. The allowed formats are zip and TextRecordInputStream. +touch +------ + +Usage: `hadoop fs -touch [-a] [-m] [-t TIMESTAMP] [-c] URI [URI ...]` + +Updates the access and modification times of the file specified by the URI to the current time. +If the file does not exist, then a zero length file is created at URI with current time as the +timestamp of that URI. + +* Use -a option to change only the access time +* Use -m option to change only the modification time +* Use -t option to specify timestamp (in format yyyyMMddHHmmss) instead of current time +* Use -c option to not create file if it does not exist + +The timestamp format is as follows +* yyyy Four digit year (e.g. 2018) +* MM Two digit month of the year (e.g. 08 for month of August) +* dd Two digit day of the month (e.g. 01 for first day of the month) +* HH Two digit hour of the day using 24 hour notation (e.g. 23 stands for 11 pm, 11 stands for 11 am) +* mm Two digit minutes of the hour +* ss Two digit seconds of the minute +e.g. 20180809230000 represents August 9th 2018, 11pm + +Example: + +* `hadoop fs -touch pathname` +* `hadoop fs -touch -m -t 20180809230000 pathname` +* `hadoop fs -touch -t 20180809230000 pathname` +* `hadoop fs -touch -a pathname` + +Exit Code: Returns 0 on success and -1 on error. + touchz ------ http://git-wip-us.apache.org/repos/asf/hadoop/blob/60ffec9f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellTouch.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellTouch.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellTouch.java index 5fe4e39..2e7cb5d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellTouch.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellTouch.java @@ -21,7 +21,11 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.junit.Assert.assertThat; +import java.text.ParseException; +import java.util.Date; + import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.shell.TouchCommands.Touch; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.StringUtils; import org.junit.Before; @@ -85,4 +89,103 @@ public class TestFsShellTouch { assertThat("Expected failed touchz in a non-existent directory", shellRun("-touchz", noDirName + "/foo"), is(not(0))); } + + @Test + public void testTouch() throws Exception { + // Ensure newFile2 does not exist + final String newFileName = "newFile2"; + final Path newFile = new Path(newFileName); + lfs.delete(newFile, true); + assertThat(lfs.exists(newFile), is(false)); + + { + assertThat( + "Expected successful touch on a non-existent file with -c option", + shellRun("-touch", "-c", newFileName), is(not(0))); + assertThat(lfs.exists(newFile), is(false)); + } + + { + String strTime = formatTimestamp(System.currentTimeMillis()); + Date dateObj = parseTimestamp(strTime); + + assertThat( + "Expected successful touch on a new file with a specified timestamp", + shellRun("-touch", "-t", strTime, newFileName), is(0)); + FileStatus new_status = lfs.getFileStatus(newFile); + assertThat(new_status.getAccessTime(), is(dateObj.getTime())); + assertThat(new_status.getModificationTime(), is(dateObj.getTime())); + } + + FileStatus fstatus = lfs.getFileStatus(newFile); + + { + String strTime = formatTimestamp(System.currentTimeMillis()); + Date dateObj = parseTimestamp(strTime); + + assertThat("Expected successful touch with a specified access time", + shellRun("-touch", "-a", "-t", strTime, newFileName), is(0)); + FileStatus new_status = lfs.getFileStatus(newFile); + // Verify if access time is recorded correctly (and modification time + // remains unchanged). + assertThat(new_status.getAccessTime(), is(dateObj.getTime())); + assertThat(new_status.getModificationTime(), + is(fstatus.getModificationTime())); + } + + fstatus = lfs.getFileStatus(newFile); + + { + String strTime = formatTimestamp(System.currentTimeMillis()); + Date dateObj = parseTimestamp(strTime); + + assertThat( + "Expected successful touch with a specified modificatiom time", + shellRun("-touch", "-m", "-t", strTime, newFileName), is(0)); + // Verify if modification time is recorded correctly (and access time + // remains unchanged). + FileStatus new_status = lfs.getFileStatus(newFile); + assertThat(new_status.getAccessTime(), is(fstatus.getAccessTime())); + assertThat(new_status.getModificationTime(), is(dateObj.getTime())); + } + + { + String strTime = formatTimestamp(System.currentTimeMillis()); + Date dateObj = parseTimestamp(strTime); + + assertThat("Expected successful touch with a specified timestamp", + shellRun("-touch", "-t", strTime, newFileName), is(0)); + + // Verify if both modification and access times are recorded correctly + FileStatus new_status = lfs.getFileStatus(newFile); + assertThat(new_status.getAccessTime(), is(dateObj.getTime())); + assertThat(new_status.getModificationTime(), is(dateObj.getTime())); + } + + { + String strTime = formatTimestamp(System.currentTimeMillis()); + Date dateObj = parseTimestamp(strTime); + + assertThat("Expected successful touch with a specified timestamp", + shellRun("-touch", "-a", "-m", "-t", strTime, newFileName), is(0)); + + // Verify if both modification and access times are recorded correctly + FileStatus new_status = lfs.getFileStatus(newFile); + assertThat(new_status.getAccessTime(), is(dateObj.getTime())); + assertThat(new_status.getModificationTime(), is(dateObj.getTime())); + } + + { + assertThat("Expected failed touch with a missing timestamp", + shellRun("-touch", "-t", newFileName), is(not(0))); + } + } + + private String formatTimestamp(long timeInMillis) { + return (new Touch()).getDateFormat().format(new Date(timeInMillis)); + } + + private Date parseTimestamp(String tstamp) throws ParseException { + return (new Touch()).getDateFormat().parse(tstamp); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/60ffec9f/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml index 6a3d53a..1798563 100644 --- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml +++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml @@ -840,6 +840,57 @@ </test> <test> <!-- TESTED --> + <description>help: help for touch</description> + <test-commands> + <command>-help touch</command> + </test-commands> + <cleanup-commands> + </cleanup-commands> + <comparators> + <comparator> + <type>RegexpComparator</type> + <expected-output>^-touch \[-a\] \[-m\] \[-t TIMESTAMP \] \[-c\] <path> \.\.\. :( )*</expected-output> + </comparator> + <comparator> + <type>RegexpComparator</type> + <expected-output>^\s*Updates the access and modification times of the file specified by the <path> to( )*</expected-output> + </comparator> + <comparator> + <type>RegexpComparator</type> + <expected-output>^\s*the current time. If the file does not exist, then a zero length file is created( )*</expected-output> + </comparator> + <comparator> + <type>RegexpComparator</type> + <expected-output>^\s*at <path> with current time as the timestamp of that <path>.( )*</expected-output> + </comparator> + <comparator> + <type>RegexpComparator</type> + <expected-output>^\s*-a\s+Change only the access time( )*</expected-output> + </comparator> + <comparator> + <type>RegexpComparator</type> + <expected-output>^\s*-a\s+Change only the access time( )*</expected-output> + </comparator> + <comparator> + <type>RegexpComparator</type> + <expected-output>^\s*-m\s+Change only the modification time( )*</expected-output> + </comparator> + <comparator> + <type>RegexpComparator</type> + <expected-output>^\s*-t\s+TIMESTAMP\s+Use specified timestamp \(in format yyyyMMddHHmmss\) instead of</expected-output> + </comparator> + <comparator> + <type>RegexpComparator</type> + <expected-output>^\s*current time( )*</expected-output> + </comparator> + <comparator> + <type>RegexpComparator</type> + <expected-output>^\s*-c\s+Do not create any files( )*</expected-output> + </comparator> + </comparators> + </test> + + <test> <!-- TESTED --> <description>help: help for touchz</description> <test-commands> <command>-help touchz</command> --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
