Updated Branches: refs/heads/trunk 6523ebb17 -> 28f415ff2
SQOOP-1261: CompilationManager should add Hadoop 2.x libraries to the classpath under Hadoop 2.x (Venkat Ranganathan via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/28f415ff Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/28f415ff Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/28f415ff Branch: refs/heads/trunk Commit: 28f415ff2f20f63e19b3c52a0e53ca566d03bac5 Parents: 6523ebb Author: Jarek Jarcec Cecho <[email protected]> Authored: Sun Dec 15 13:57:15 2013 -0800 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Sun Dec 15 13:57:15 2013 -0800 ---------------------------------------------------------------------- .../apache/sqoop/orm/CompilationManager.java | 36 +++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/28f415ff/src/java/org/apache/sqoop/orm/CompilationManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/orm/CompilationManager.java b/src/java/org/apache/sqoop/orm/CompilationManager.java index 371ecab..d56ed60 100644 --- a/src/java/org/apache/sqoop/orm/CompilationManager.java +++ b/src/java/org/apache/sqoop/orm/CompilationManager.java @@ -24,6 +24,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.jar.JarOutputStream; import java.util.zip.ZipEntry; @@ -77,7 +78,7 @@ public class CompilationManager { * If that doesn't work, check our classpath. * @return the filename of the hadoop-*-core.jar file. */ - private String findHadoopCoreJar() { + private String findHadoopJars() { String hadoopMapRedHome = options.getHadoopMapRedHome(); if (null == hadoopMapRedHome) { @@ -91,22 +92,30 @@ public class CompilationManager { File hadoopMapRedHomeFile = new File(hadoopMapRedHome); LOG.info("HADOOP_MAPRED_HOME is " + hadoopMapRedHomeFile.getAbsolutePath()); - File [] entries = hadoopMapRedHomeFile.listFiles(); + + Iterator<File> filesIterator = FileUtils.iterateFiles(hadoopMapRedHomeFile, + new String[] { "jar" }, true); + StringBuilder sb = new StringBuilder(); + + while (filesIterator.hasNext()) { + File file = filesIterator.next(); + String name = file.getName(); + if (name.startsWith("hadoop-common") + || name.startsWith("hadoop-mapreduce-client-core") + || name.startsWith("hadoop-core")) { + sb.append(file.getAbsolutePath()); + sb.append(File.pathSeparator); + } + } - if (null == entries) { + if (sb.length() < 1) { LOG.warn("HADOOP_MAPRED_HOME appears empty or missing"); return Jars.getJarPathForClass(JobConf.class); } - for (File f : entries) { - if (f.getName().startsWith("hadoop-") - && f.getName().endsWith("-core.jar")) { - LOG.info("Found hadoop core jar at: " + f.getAbsolutePath()); - return f.getAbsolutePath(); - } - } - - return Jars.getJarPathForClass(JobConf.class); + String s = sb.substring(0, sb.length() - 1); + LOG.debug("Returning jar file path " + s); + return s; } /** @@ -134,7 +143,7 @@ public class CompilationManager { } // find hadoop-*-core.jar for classpath. - String coreJar = findHadoopCoreJar(); + String coreJar = findHadoopJars(); if (null == coreJar) { // Couldn't find a core jar to insert into the CP for compilation. If, // however, we're running this from a unit test, then the path to the @@ -159,6 +168,7 @@ public class CompilationManager { } String curClasspath = System.getProperty("java.class.path"); + LOG.debug("Current sqoop classpath = " + curClasspath); args.add("-sourcepath"); args.add(jarOutDir);
