HIVE-16076 : LLAP packaging - include aux libs (Sergey Shelukhin, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/039fee7f Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/039fee7f Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/039fee7f Branch: refs/heads/hive-14535 Commit: 039fee7f944199eece949e24951e1f7be9c879d4 Parents: 9d3ff49 Author: Sergey Shelukhin <[email protected]> Authored: Thu Mar 9 17:19:23 2017 -0800 Committer: Sergey Shelukhin <[email protected]> Committed: Thu Mar 9 17:19:23 2017 -0800 ---------------------------------------------------------------------- .../hive/llap/cli/LlapOptionsProcessor.java | 22 ++++++++++++++---- .../hadoop/hive/llap/cli/LlapServiceDriver.java | 24 ++++++++++++++++---- 2 files changed, 37 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/039fee7f/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java index 6463b30..aa09083 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java @@ -48,6 +48,7 @@ public class LlapOptionsProcessor { public static final String OPTION_SIZE = "size"; // forward via config.json public static final String OPTION_XMX = "xmx"; // forward as arg public static final String OPTION_AUXJARS = "auxjars"; // used to localize jars + public static final String OPTION_AUXHIVE = "auxhive"; // used to localize jars public static final String OPTION_AUXHBASE = "auxhbase"; // used to localize jars public static final String OPTION_JAVA_HOME = "javaHome"; // forward via config.json public static final String OPTION_HIVECONF = "hiveconf"; // llap-daemon-site if relevant parameter @@ -86,11 +87,12 @@ public class LlapOptionsProcessor { private final String logger; private final boolean isStarting; private final String output; + private final boolean isHiveAux; public LlapOptions(String name, int instances, String directory, int executors, int ioThreads, - long cache, long size, long xmx, String jars, boolean isHbase, - @Nonnull Properties hiveconf, String javaPath, String llapQueueName, - String logger, boolean isStarting, String output) throws ParseException { + long cache, long size, long xmx, String jars, boolean isHbase, + @Nonnull Properties hiveconf, String javaPath, String llapQueueName, String logger, + boolean isStarting, String output, boolean isHiveAux) throws ParseException { if (instances <= 0) { throw new ParseException("Invalid configuration: " + instances + " (should be greater than 0)"); @@ -105,6 +107,7 @@ public class LlapOptionsProcessor { this.xmx = xmx; this.jars = jars; this.isHbase = isHbase; + this.isHiveAux = isHiveAux; this.conf = hiveconf; this.javaPath = javaPath; this.llapQueueName = llapQueueName; @@ -157,6 +160,10 @@ public class LlapOptionsProcessor { return isHbase; } + public boolean getIsHiveAux() { + return isHiveAux; + } + public Properties getConfig() { return conf; } @@ -253,6 +260,9 @@ public class LlapOptionsProcessor { options.addOption(OptionBuilder.hasArg().withArgName(OPTION_AUXHBASE).withLongOpt(OPTION_AUXHBASE) .withDescription("whether to package the HBase jars (true by default)").create('h')); + options.addOption(OptionBuilder.hasArg().withArgName(OPTION_AUXHIVE).withLongOpt(OPTION_AUXHIVE) + .withDescription("whether to package the Hive aux jars (true by default)").create(OPTION_AUXHIVE)); + options.addOption(OptionBuilder.hasArg().withArgName(OPTION_JAVA_HOME).withLongOpt(OPTION_JAVA_HOME) .withDescription( "Path to the JRE/JDK. This should be installed at the same location on all cluster nodes ($JAVA_HOME, java.home by default)") @@ -310,6 +320,8 @@ public class LlapOptionsProcessor { final long xmx = parseSuffixed(commandLine.getOptionValue(OPTION_XMX, "-1")); final boolean isHbase = Boolean.parseBoolean( commandLine.getOptionValue(OPTION_AUXHBASE, "true")); + final boolean isHiveAux = Boolean.parseBoolean( + commandLine.getOptionValue(OPTION_AUXHIVE, "true")); final boolean doStart = commandLine.hasOption(OPTION_START); final String output = commandLine.getOptionValue(OPTION_OUTPUT_DIR, null); @@ -340,8 +352,8 @@ public class LlapOptionsProcessor { // loglevel, chaosmonkey & args are parsed by the python processor - return new LlapOptions(name, instances, directory, executors, ioThreads, cache, - size, xmx, jars, isHbase, hiveconf, javaHome, queueName, logger, doStart, output); + return new LlapOptions(name, instances, directory, executors, ioThreads, cache, size, xmx, + jars, isHbase, hiveconf, javaHome, queueName, logger, doStart, output, isHiveAux); } private void printUsage() { http://git-wip-us.apache.org/repos/asf/hive/blob/039fee7f/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java index e8517ab..22e5ee8 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java @@ -102,7 +102,7 @@ public class LlapServiceDriver { * This is a working configuration for the instance to merge various variables. * It is not written out for llap server usage */ - private final Configuration conf; + private final HiveConf conf; public LlapServiceDriver() { SessionState ss = SessionState.get(); @@ -435,17 +435,33 @@ public class LlapServiceDriver { } } - String auxJars = options.getAuxJars(); + HashSet<String> auxJars = new HashSet<>(); + // There are many ways to have AUX jars in Hive... sigh + if (options.getIsHiveAux()) { + // Note: we don't add ADDED jars, RELOADABLE jars, etc. That is by design; there are too many ways + // to add jars in Hive, some of which are session/etc. specific. Env + conf + arg should be enough. + addAuxJarsToSet(auxJars, conf.getAuxJars()); + addAuxJarsToSet(auxJars, System.getenv("HIVE_AUX_JARS_PATH")); + LOG.info("Adding the following aux jars from the environment and configs: " + auxJars); + } + + addAuxJarsToSet(auxJars, options.getAuxJars()); + for (String jarPath : auxJars) { + lfs.copyFromLocalFile(new Path(jarPath), libDir); + } + return null; + } + + private void addAuxJarsToSet(HashSet<String> auxJarSet, String auxJars) { if (auxJars != null && !auxJars.isEmpty()) { // TODO: transitive dependencies warning? String[] jarPaths = auxJars.split(","); for (String jarPath : jarPaths) { if (!jarPath.isEmpty()) { - lfs.copyFromLocalFile(new Path(jarPath), libDir); + auxJarSet.add(jarPath); } } } - return null; } };
