Updated Branches: refs/heads/master 092ef5d01 -> 4a3aa0dfd
CRUNCH-131: Use globStatus in SourceTargetHelper so that wildcard patterns work properly with Crunch. Contributed by Dave Beech. Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/4a3aa0df Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/4a3aa0df Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/4a3aa0df Branch: refs/heads/master Commit: 4a3aa0dfd9dbdfd2532b03776d87b99f71fae5a5 Parents: 092ef5d Author: Josh Wills <[email protected]> Authored: Thu Dec 13 11:15:50 2012 -0800 Committer: Josh Wills <[email protected]> Committed: Thu Dec 13 11:15:50 2012 -0800 ---------------------------------------------------------------------- .../org/apache/crunch/io/SourceTargetHelper.java | 22 ++------------- 1 files changed, 3 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/4a3aa0df/crunch/src/main/java/org/apache/crunch/io/SourceTargetHelper.java ---------------------------------------------------------------------- diff --git a/crunch/src/main/java/org/apache/crunch/io/SourceTargetHelper.java b/crunch/src/main/java/org/apache/crunch/io/SourceTargetHelper.java index bbe5eaa..65fb35a 100644 --- a/crunch/src/main/java/org/apache/crunch/io/SourceTargetHelper.java +++ b/crunch/src/main/java/org/apache/crunch/io/SourceTargetHelper.java @@ -17,7 +17,6 @@ */ package org.apache.crunch.io; -import java.io.FileNotFoundException; import java.io.IOException; import org.apache.commons.logging.Log; @@ -40,28 +39,13 @@ public class SourceTargetHelper { } public static long getPathSize(FileSystem fs, Path path) throws IOException { - - if (!fs.exists(path)) { + FileStatus[] stati = fs.globStatus(path); + if (stati == null || stati.length == 0) { return -1L; } - - FileStatus[] stati = null; - try { - stati = fs.listStatus(path); - if (stati == null) { - throw new FileNotFoundException(path + " doesn't exist"); - } - } catch (FileNotFoundException e) { - LOG.warn("Returning 0 for getPathSize on non-existant path '" + path + "'"); - return 0L; - } - - if (stati.length == 0) { - return 0L; - } long size = 0; for (FileStatus status : stati) { - size += status.getLen(); + size += fs.getContentSummary(status.getPath()).getLength(); } return size; }
