HDFS-206. Support for head in FSShell. Contributed by Gabor Bota.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/81225430 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/81225430 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/81225430 Branch: refs/heads/YARN-1011 Commit: 81225430d0153e8515f2ecb56ece7fcfe289d5ee Parents: 2fae63a Author: Sean Mackrory <mackror...@apache.org> Authored: Mon Oct 30 16:35:42 2017 -0600 Committer: Sean Mackrory <mackror...@apache.org> Committed: Tue Oct 31 08:34:01 2017 -0600 ---------------------------------------------------------------------- .../org/apache/hadoop/fs/shell/FsCommand.java | 1 + .../java/org/apache/hadoop/fs/shell/Head.java | 78 ++++++++++++++++++++ .../src/site/markdown/FileSystemShell.md | 13 ++++ .../org/apache/hadoop/hdfs/TestDFSShell.java | 27 +++++++ 4 files changed, 119 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/81225430/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java index f274f67..4a13414 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java @@ -64,6 +64,7 @@ abstract public class FsCommand extends Command { factory.registerCommands(SetReplication.class); factory.registerCommands(Stat.class); factory.registerCommands(Tail.class); + factory.registerCommands(Head.class); factory.registerCommands(Test.class); factory.registerCommands(Touch.class); factory.registerCommands(Truncate.class); http://git-wip-us.apache.org/repos/asf/hadoop/blob/81225430/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Head.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Head.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Head.java new file mode 100644 index 0000000..2280225 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Head.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.shell; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.PathIsDirectoryException; +import org.apache.hadoop.io.IOUtils; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; + +/** + * Show the first 1KB of the file. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable + +class Head extends FsCommand { + public static void registerCommands(CommandFactory factory) { + factory.addClass(Head.class, "-head"); + } + public static final String NAME = "head"; + public static final String USAGE = "<file>"; + public static final String DESCRIPTION = + "Show the first 1KB of the file.\n"; + + private long endingOffset = 1024; + + @Override + protected void processOptions(LinkedList<String> args) throws IOException { + CommandFormat cf = new CommandFormat(1, 1); + cf.parse(args); + } + + @Override + protected List<PathData> expandArgument(String arg) throws IOException { + List<PathData> items = new LinkedList<PathData>(); + items.add(new PathData(arg, getConf())); + return items; + } + + @Override + protected void processPath(PathData item) throws IOException { + if (item.stat.isDirectory()) { + throw new PathIsDirectoryException(item.toString()); + } + + dumpToOffset(item); + } + + private void dumpToOffset(PathData item) throws IOException { + FSDataInputStream in = item.fs.open(item.path); + try { + IOUtils.copyBytes(in, System.out, endingOffset, false); + } finally { + in.close(); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/81225430/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md index 71eec75..ec9d3c3 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md @@ -400,6 +400,19 @@ Exit Code: Returns 0 on success and non-zero on error. +head +---- + +Usage: `hadoop fs -head URI` + +Displays first kilobyte of the file to stdout. + +Example: + +* `hadoop fs -head pathname` + +Exit Code: Returns 0 on success and -1 on error. + help ---- http://git-wip-us.apache.org/repos/asf/hadoop/blob/81225430/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java index 9ae49aa..e82863a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java @@ -893,6 +893,33 @@ public class TestDFSShell { } /** + * Test that -head displays first kilobyte of the file to stdout. + */ + @Test (timeout = 30000) + public void testHead() throws Exception { + final int fileLen = 5 * BLOCK_SIZE; + + // create a text file with multiple KB bytes (and multiple blocks) + final Path testFile = new Path("testHead", "file1"); + final String text = RandomStringUtils.randomAscii(fileLen); + try (OutputStream pout = dfs.create(testFile)) { + pout.write(text.getBytes()); + } + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + System.setOut(new PrintStream(out)); + final String[] argv = new String[]{"-head", testFile.toString()}; + final int ret = ToolRunner.run(new FsShell(dfs.getConf()), argv); + + assertEquals(Arrays.toString(argv) + " returned " + ret, 0, ret); + assertEquals("-head returned " + out.size() + " bytes data, expected 1KB", + 1024, out.size()); + // tailed out last 1KB of the file content + assertArrayEquals("Head output doesn't match input", + text.substring(0, 1024).getBytes(), out.toByteArray()); + out.reset(); + } + + /** * Test that -tail displays last kilobyte of the file to stdout. */ @Test (timeout = 30000) --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
