Repository: hive Updated Branches: refs/heads/master a42314deb -> bff9da2cc
HIVE-18016 : org.apache.hadoop.hive.ql.util.ResourceDownloader - Review (Beluga Behr via Ashutosh Chauhan) Signed-off-by: Ashutosh Chauhan <hashut...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bff9da2c Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bff9da2c Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bff9da2c Branch: refs/heads/master Commit: bff9da2cc03da848189c7266ee57069dde3fe668 Parents: a42314d Author: BELUGA BEHR <dam6...@gmail.com> Authored: Wed Nov 8 16:04:19 2017 -0800 Committer: Ashutosh Chauhan <hashut...@apache.org> Committed: Wed Nov 8 16:04:19 2017 -0800 ---------------------------------------------------------------------- .../hadoop/hive/ql/util/ResourceDownloader.java | 42 ++++++++++---------- 1 file changed, 20 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/bff9da2c/ql/src/java/org/apache/hadoop/hive/ql/util/ResourceDownloader.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/util/ResourceDownloader.java b/ql/src/java/org/apache/hadoop/hive/ql/util/ResourceDownloader.java index 42ed302..9fa1ea1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/util/ResourceDownloader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/util/ResourceDownloader.java @@ -22,18 +22,18 @@ import java.io.File; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.util.Collections; import java.util.List; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.util.Shell; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.collect.Lists; - public class ResourceDownloader { private static final Logger LOG = LoggerFactory.getLogger(ResourceDownloader.class); private final DependencyResolver dependencyResolver; @@ -56,21 +56,19 @@ public class ResourceDownloader { } public static boolean isIvyUri(String value) throws URISyntaxException { - return "ivy".equalsIgnoreCase(createURI(value).getScheme()); + return UriType.IVY == getURLType(createURI(value)); } public static boolean isHdfsUri(String value) throws URISyntaxException { - return "hdfs".equalsIgnoreCase(createURI(value).getScheme()); + return UriType.HDFS == getURLType(createURI(value)); } public static boolean isFileUri(String value) { - String scheme = null; try { - scheme = createURI(value).getScheme(); + return UriType.FILE == getURLType(createURI(value)); } catch (URISyntaxException ex) { throw new RuntimeException(ex); } - return (scheme == null) || scheme.equalsIgnoreCase("file"); } public List<URI> resolveAndDownload(String source, boolean convertToUnix) @@ -86,19 +84,18 @@ public class ResourceDownloader { private List<URI> resolveAndDownloadInternal(URI source, String subDir, boolean convertToUnix, boolean isLocalAllowed) throws URISyntaxException, IOException { switch (getURLType(source)) { - case FILE: return isLocalAllowed ? Lists.newArrayList(source) : null; + case FILE: return isLocalAllowed ? Collections.singletonList(source) : null; case IVY: return dependencyResolver.downloadDependencies(source); case HDFS: case OTHER: - return Lists.newArrayList( - createURI(downloadResource(source, subDir, convertToUnix))); + return Collections.singletonList(createURI(downloadResource(source, subDir, convertToUnix))); default: throw new AssertionError(getURLType(source)); } } private String downloadResource(URI srcUri, String subDir, boolean convertToUnix) throws IOException, URISyntaxException { - LOG.info("converting to local " + srcUri); + LOG.info("converting to local {}", srcUri); File destinationDir = (subDir == null) ? resourceDir : new File(resourceDir, subDir); ensureDirectory(destinationDir); File destinationFile = new File(destinationDir, new Path(srcUri.toString()).getName()); @@ -114,23 +111,24 @@ public class ResourceDownloader { } private static void ensureDirectory(File resourceDir) { - boolean doesExist = resourceDir.exists(); - if (doesExist && !resourceDir.isDirectory()) { - throw new RuntimeException(resourceDir + " is not a directory"); - } - if (!doesExist && !resourceDir.mkdirs()) { - throw new RuntimeException("Couldn't create directory " + resourceDir); + try { + FileUtils.forceMkdir(resourceDir); + } catch (IOException e) { + throw new RuntimeException(e); } } private enum UriType { IVY, FILE, HDFS, OTHER }; - private static ResourceDownloader.UriType getURLType(URI value) throws URISyntaxException { - String scheme = value.getScheme(); + + /** + * If the URI has no scheme defined, the default is {@link UriType#FILE} + */ + private static ResourceDownloader.UriType getURLType(URI value) { + String scheme = StringUtils.lowerCase(value.getScheme()); if (scheme == null) return UriType.FILE; - scheme = scheme.toLowerCase(); - if ("ivy".equals(scheme)) return UriType.IVY; if ("file".equals(scheme)) return UriType.FILE; if ("hdfs".equals(scheme)) return UriType.HDFS; + if ("ivy".equals(scheme)) return UriType.IVY; return UriType.OTHER; } }