Repository: hadoop Updated Branches: refs/heads/trunk 6ba8fd7e8 -> 9937eef7f
HADOOP-11421. Add IOUtils#listDirectory (cmccabe) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9937eef7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9937eef7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9937eef7 Branch: refs/heads/trunk Commit: 9937eef7f7f04a7dd3d504ae7ec5852d488a1f6a Parents: 6ba8fd7 Author: Colin Patrick Mccabe <[email protected]> Authored: Wed Dec 17 10:31:19 2014 -0800 Committer: Colin Patrick Mccabe <[email protected]> Committed: Wed Dec 17 15:18:16 2014 -0800 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 2 + .../main/java/org/apache/hadoop/io/IOUtils.java | 36 ++++++++++++++++ .../java/org/apache/hadoop/io/TestIOUtils.java | 44 ++++++++++++++++++++ 3 files changed, 82 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9937eef7/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 049dcf9..bd0f5b5 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -427,6 +427,8 @@ Release 2.7.0 - UNRELEASED HADOOP-11248. Add hadoop configuration to disable Azure Filesystem metrics collection. (Shanyu Zhao via cnauroth) + HADOOP-11421. Add IOUtils#listDirectory (cmccabe) + OPTIMIZATIONS HADOOP-11323. WritableComparator#compare keeps reference to byte array. http://git-wip-us.apache.org/repos/asf/hadoop/blob/9937eef7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java index 6be7446..e6c00c9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java @@ -23,6 +23,12 @@ import java.net.Socket; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.channels.WritableByteChannel; +import java.nio.file.DirectoryStream; +import java.nio.file.DirectoryIteratorException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -30,6 +36,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ChunkedArrayList; /** * An utility class for I/O related functionality. @@ -313,4 +320,33 @@ public class IOUtils { offset += fc.write(buf, offset); } while (buf.remaining() > 0); } + + /** + * Return the complete list of files in a directory as strings.<p/> + * + * This is better than File#listDir because it does not ignore IOExceptions. + * + * @param dir The directory to list. + * @param filter If non-null, the filter to use when listing + * this directory. + * @return The list of files in the directory. + * + * @throws IOException On I/O error + */ + public static List<String> listDirectory(File dir, FilenameFilter filter) + throws IOException { + ArrayList<String> list = new ArrayList<String> (); + try (DirectoryStream<Path> stream = + Files.newDirectoryStream(dir.toPath())) { + for (Path entry: stream) { + String fileName = entry.getFileName().toString(); + if ((filter == null) || filter.accept(dir, fileName)) { + list.add(fileName); + } + } + } catch (DirectoryIteratorException e) { + throw e.getCause(); + } + return list; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9937eef7/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java index 5db3c91..ac4fb48 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java @@ -24,14 +24,21 @@ import static org.junit.Assert.fail; import java.io.ByteArrayInputStream; import java.io.EOFException; import java.io.File; +import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.apache.commons.io.FileUtils; import org.apache.hadoop.test.GenericTestUtils; +import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; @@ -245,4 +252,41 @@ public class TestIOUtils { in.close(); } } + + private static enum NoEntry3Filter implements FilenameFilter { + INSTANCE; + + @Override + public boolean accept(File dir, String name) { + return !name.equals("entry3"); + } + } + + @Test + public void testListDirectory() throws IOException { + File dir = new File("testListDirectory"); + Files.createDirectory(dir.toPath()); + try { + Set<String> entries = new HashSet<String>(); + entries.add("entry1"); + entries.add("entry2"); + entries.add("entry3"); + for (String entry : entries) { + Files.createDirectory(new File(dir, entry).toPath()); + } + List<String> list = IOUtils.listDirectory(dir, + NoEntry3Filter.INSTANCE); + for (String entry : list) { + Assert.assertTrue(entries.remove(entry)); + } + Assert.assertTrue(entries.contains("entry3")); + list = IOUtils.listDirectory(dir, null); + for (String entry : list) { + entries.remove(entry); + } + Assert.assertTrue(entries.isEmpty()); + } finally { + FileUtils.deleteDirectory(dir); + } + } }
