Author: ekoifman
Date: Thu Nov  6 20:20:31 2014
New Revision: 1637207

URL: http://svn.apache.org/r1637207
Log:
HIVE-8754 Sqoop job submission via WebHCat doesn't properly localize required 
jdbc jars in secure cluster (Eugene Koifman, reviewed by Thejas Nair)

Modified:
    
hive/branches/branch-0.14/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java
    
hive/branches/branch-0.14/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java

Modified: 
hive/branches/branch-0.14/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java
URL: 
http://svn.apache.org/viewvc/hive/branches/branch-0.14/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java?rev=1637207&r1=1637206&r2=1637207&view=diff
==============================================================================
--- 
hive/branches/branch-0.14/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java
 (original)
+++ 
hive/branches/branch-0.14/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java
 Thu Nov  6 20:20:31 2014
@@ -83,6 +83,20 @@ public class SqoopDelegator extends Laun
             args.add("-D" + TempletonControllerJob.TOKEN_FILE_ARG_PLACEHOLDER);
             args.add("-D" + 
TempletonControllerJob.MAPREDUCE_JOB_TAGS_ARG_PLACEHOLDER);
           }
+          if(i == 0 && TempletonUtils.isset(libdir) && 
TempletonUtils.isset(appConf.sqoopArchive())) {
+            
//http://sqoop.apache.org/docs/1.4.5/SqoopUserGuide.html#_using_generic_and_specific_arguments
+            String libJars = null;
+            for(String s : args) {
+              if(s.startsWith(JobSubmissionConstants.Sqoop.LIB_JARS)) {
+                libJars = s.substring(s.indexOf("=") + 1);
+                break;
+              }
+            }
+            //the jars in libJars will be localized to CWD of the launcher 
task; then -libjars will
+            //cause them to be localized for the Sqoop MR job tasks
+            args.add(TempletonUtils.quoteForWindows("-libjars"));
+            args.add(TempletonUtils.quoteForWindows(libJars));
+          }
         }
       } else if (TempletonUtils.isset(optionsFile)) {
         args.add("--options-file");
@@ -114,11 +128,13 @@ public class SqoopDelegator extends Laun
       /**Sqoop accesses databases via JDBC.  This means it needs to have 
appropriate JDBC
       drivers available.  Normally, the user would install Sqoop and place 
these jars
       into SQOOP_HOME/lib.  When WebHCat is configured to auto-ship the Sqoop 
tar file, we
-      need to make sure that relevant JDBC jars are available on target node.
+      need to make sure that relevant JDBC jars are available on target node 
but we cannot modify
+      lib/ of exploded tar because Dist Cache intentionally prevents this.
       The user is expected to place any JDBC jars into an HDFS directory and 
specify this
-      dir in "libdir" parameter.  All the files in this dir will be copied to 
lib/ of the
-      exploded Sqoop tar ball on target node.
+      dir in "libdir" parameter.  WebHCat then ensures that these jars are 
localized for the launcher task
+      and made available to Sqoop.
       {@link 
org.apache.hive.hcatalog.templeton.tool.LaunchMapper#handleSqoop(org.apache.hadoop.conf.Configuration,
 java.util.Map)}
+      {@link #makeArgs(String, String, String, String, String, boolean, 
String)}
       */
       LOG.debug("libdir=" + libdir);
       List<Path> jarList = TempletonUtils.hadoopFsListChildren(libdir, 
appConf, runAs);

Modified: 
hive/branches/branch-0.14/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
URL: 
http://svn.apache.org/viewvc/hive/branches/branch-0.14/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java?rev=1637207&r1=1637206&r2=1637207&view=diff
==============================================================================
--- 
hive/branches/branch-0.14/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
 (original)
+++ 
hive/branches/branch-0.14/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
 Thu Nov  6 20:20:31 2014
@@ -18,7 +18,6 @@
  */
 package org.apache.hive.hcatalog.templeton.tool;
 
-import com.google.common.io.Files;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -101,11 +100,18 @@ public class LaunchMapper extends Mapper
     if(TempletonUtils.isset(conf.get(Sqoop.LIB_JARS))) {
       //LIB_JARS should only be set if Sqoop is auto-shipped
       LOG.debug(Sqoop.LIB_JARS + "=" + conf.get(Sqoop.LIB_JARS));
-      //copy these (which have now been localized) jars to sqoop/lib
-      String destDir = conf.get(AppConfig.SQOOP_HOME_PATH) + File.separator + 
"lib";
       String[] files = conf.getStrings(Sqoop.LIB_JARS);
+      StringBuilder jdbcJars = new StringBuilder();
       for(String f : files) {
-        Files.copy(new File(f), new File(destDir + File.separator + f));
+        jdbcJars.append(f).append(File.pathSeparator);
+      }
+      jdbcJars.setLength(jdbcJars.length() - 1);
+      //this makes the jars available to Sqoop client
+      if(TempletonUtils.isset(System.getenv("HADOOP_CLASSPATH"))) {
+        env.put("HADOOP_CLASSPATH", System.getenv("HADOOP_CLASSPATH") + 
File.pathSeparator + jdbcJars.toString());
+      }
+      else {
+        env.put("HADOOP_CLASSPATH", jdbcJars.toString());
       }
     }
   }


Reply via email to