Repository: hive Updated Branches: refs/heads/branch-3.1 f9cfbd9d6 -> 11ddb8bea
HIVE-19978: Backport HIVE-18037 to branch-3 (Daniel Voros via Zoltan Haindrich) Signed-off-by: Zoltan Haindrich <k...@rxd.hu> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/11ddb8be Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/11ddb8be Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/11ddb8be Branch: refs/heads/branch-3.1 Commit: 11ddb8beae56c74e962057d6540c7478987c7c06 Parents: f9cfbd9 Author: Daniel Voros <daniel.vo...@gmail.com> Authored: Thu Jun 28 12:58:04 2018 +0200 Committer: Vineet Garg <vg...@apache.org> Committed: Thu Jun 28 10:22:40 2018 -0700 ---------------------------------------------------------------------- bin/ext/llap.sh | 4 +- binary-package-licenses/README | 1 - .../org/apache/hadoop/hive/conf/HiveConf.java | 6 +- jdbc/pom.xml | 1 - .../hive/llap/registry/LlapServiceInstance.java | 2 +- llap-server/bin/llapDaemon.sh | 2 +- llap-server/changes_for_non_slider_install.txt | 2 +- llap-server/pom.xml | 83 +---- .../hive/llap/cli/LlapOptionsProcessor.java | 50 ++- .../hadoop/hive/llap/cli/LlapServiceDriver.java | 11 +- .../hadoop/hive/llap/cli/LlapSliderUtils.java | 211 ++++------- .../hive/llap/cli/LlapStatusServiceDriver.java | 353 +++++-------------- .../hadoop/hive/llap/daemon/impl/QueryInfo.java | 2 +- llap-server/src/main/resources/llap.py | 76 ---- llap-server/src/main/resources/package.py | 115 +++--- llap-server/src/main/resources/params.py | 40 --- llap-server/src/main/resources/templates.py | 179 +++------- packaging/src/main/assembly/bin.xml | 2 +- pom.xml | 1 - 19 files changed, 302 insertions(+), 839 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/11ddb8be/bin/ext/llap.sh ---------------------------------------------------------------------- diff --git a/bin/ext/llap.sh b/bin/ext/llap.sh index 0462d26..91a54b3 100644 --- a/bin/ext/llap.sh +++ b/bin/ext/llap.sh @@ -17,7 +17,7 @@ THISSERVICE=llap export SERVICE_LIST="${SERVICE_LIST}${THISSERVICE} " llap () { - TMPDIR=$(mktemp -d /tmp/staging-slider-XXXXXX) + TMPDIR=$(mktemp -d /tmp/staging-yarn-XXXXXX) CLASS=org.apache.hadoop.hive.llap.cli.LlapServiceDriver; if [ ! -f ${HIVE_LIB}/hive-cli-*.jar ]; then echo "Missing Hive CLI Jar" @@ -37,7 +37,7 @@ llap () { # check for config files test -f $TMPDIR/config.json - python $HIVE_HOME/scripts/llap/slider/package.py --input $TMPDIR "$@" + python $HIVE_HOME/scripts/llap/yarn/package.py --input $TMPDIR "$@" # remove temp files rm -rf $TMPDIR http://git-wip-us.apache.org/repos/asf/hive/blob/11ddb8be/binary-package-licenses/README ---------------------------------------------------------------------- diff --git a/binary-package-licenses/README b/binary-package-licenses/README index c801896..db6b7fe 100644 --- a/binary-package-licenses/README +++ b/binary-package-licenses/README @@ -74,7 +74,6 @@ pentaho-aggdesigner-algorithm plexus-utils regexp server-metrics -slider-core snappy* stax-api super-csv http://git-wip-us.apache.org/repos/asf/hive/blob/11ddb8be/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 2934052..e7c3c0a 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -3961,15 +3961,15 @@ public class HiveConf extends Configuration { "Number of RPC handlers for AM LLAP plugin endpoint."), LLAP_DAEMON_WORK_DIRS("hive.llap.daemon.work.dirs", "", "Working directories for the daemon. This should not be set if running as a YARN\n" + - "application via Slider. It must be set when not running via Slider on YARN. If the value\n" + - "is set when running as a Slider YARN application, the specified value will be used.", + "Service. It must be set when not running on YARN. If the value is set when\n" + + "running as a YARN Service, the specified value will be used.", "llap.daemon.work.dirs"), LLAP_DAEMON_YARN_SHUFFLE_PORT("hive.llap.daemon.yarn.shuffle.port", 15551, "YARN shuffle port for LLAP-daemon-hosted shuffle.", "llap.daemon.yarn.shuffle.port"), LLAP_DAEMON_YARN_CONTAINER_MB("hive.llap.daemon.yarn.container.mb", -1, "llap server yarn container size in MB. Used in LlapServiceDriver and package.py", "llap.daemon.yarn.container.mb"), LLAP_DAEMON_QUEUE_NAME("hive.llap.daemon.queue.name", null, - "Queue name within which the llap slider application will run." + + "Queue name within which the llap application will run." + " Used in LlapServiceDriver and package.py"), // TODO Move the following 2 properties out of Configuration to a constant. LLAP_DAEMON_CONTAINER_ID("hive.llap.daemon.container.id", null, http://git-wip-us.apache.org/repos/asf/hive/blob/11ddb8be/jdbc/pom.xml ---------------------------------------------------------------------- diff --git a/jdbc/pom.xml b/jdbc/pom.xml index 81f84ce..6413894 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -304,7 +304,6 @@ <exclude>org.apache.directory.server:*</exclude> <exclude>org.apache.geronimo.specs:*</exclude> <exclude>org.apache.htrace:*</exclude> - <exclude>org.apache.slider:*</exclude> <exclude>org.codehaus.jackson:*</exclude> <exclude>org.codehaus.jettison:*</exclude> <exclude>org.datanucleus:*</exclude> http://git-wip-us.apache.org/repos/asf/hive/blob/11ddb8be/llap-client/src/java/org/apache/hadoop/hive/llap/registry/LlapServiceInstance.java ---------------------------------------------------------------------- diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/LlapServiceInstance.java b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/LlapServiceInstance.java index 30b1810..942f1d0 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/LlapServiceInstance.java +++ b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/LlapServiceInstance.java @@ -51,7 +51,7 @@ public interface LlapServiceInstance extends ServiceInstance { /** * Memory and Executors available for the LLAP tasks * - * This does not include the size of the cache or the actual vCores allocated via Slider. + * This does not include the size of the cache or the actual vCores allocated via YARN. * * @return */ http://git-wip-us.apache.org/repos/asf/hive/blob/11ddb8be/llap-server/bin/llapDaemon.sh ---------------------------------------------------------------------- diff --git a/llap-server/bin/llapDaemon.sh b/llap-server/bin/llapDaemon.sh index 4945473..58188e2 100755 --- a/llap-server/bin/llapDaemon.sh +++ b/llap-server/bin/llapDaemon.sh @@ -113,7 +113,7 @@ case $startStop in #rotate_log $logOut echo starting llapdaemon, logging to $logLog and $logOut export LLAP_DAEMON_LOGFILE=${LLAP_DAEMON_LOG_BASE}.log - nohup nice -n $LLAP_DAEMON_NICENESS "$LLAP_DAEMON_BIN_HOME"/runLlapDaemon.sh run >> "$logOut" 2>&1 < /dev/null & + nohup nice -n $LLAP_DAEMON_NICENESS "$LLAP_DAEMON_BIN_HOME"/runLlapDaemon.sh run >> "$logOut" 2>&1 < /dev/null echo $! > $pid # capture the ulimit output echo "ulimit -a" >> $logOut http://git-wip-us.apache.org/repos/asf/hive/blob/11ddb8be/llap-server/changes_for_non_slider_install.txt ---------------------------------------------------------------------- diff --git a/llap-server/changes_for_non_slider_install.txt b/llap-server/changes_for_non_slider_install.txt index ec20fe1..afcae31 100644 --- a/llap-server/changes_for_non_slider_install.txt +++ b/llap-server/changes_for_non_slider_install.txt @@ -1,4 +1,4 @@ --- Incomplete list of changes required to run LLAP as standalone daemons outside of YARN, instead of as a Slider App +-- Incomplete list of changes required to run LLAP as standalone daemons outside of YARN, instead of as a YARN Service hive.llap.daemon.work.dirs - Create dirs on each node, and set to this value. hive.llap.daemon.service.hosts - Comma separated list of hosts on which llap will run. ApplicationConstants.Environment.CONTAINER_ID should be available in the environment for secure clusters http://git-wip-us.apache.org/repos/asf/hive/blob/11ddb8be/llap-server/pom.xml ---------------------------------------------------------------------- diff --git a/llap-server/pom.xml b/llap-server/pom.xml index 4405700..d3a0694 100644 --- a/llap-server/pom.xml +++ b/llap-server/pom.xml @@ -122,6 +122,22 @@ </dependency> <dependency> <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-services-core</artifactId> + <version>${hadoop.version}</version> + <optional>true</optional> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + <exclusion> + <groupId>commmons-logging</groupId> + <artifactId>commons-logging</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> <version>${hadoop.version}</version> <optional>true</optional> @@ -202,73 +218,6 @@ </exclusions> </dependency> <dependency> - <groupId>org.apache.slider</groupId> - <artifactId>slider-core</artifactId> - <version>${slider.version}</version> - <exclusions> - <exclusion> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jetty</artifactId> - </exclusion> - <exclusion> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jetty-util</artifactId> - </exclusion> - <exclusion> - <groupId>asm</groupId> - <artifactId>asm</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-client</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-server-web-proxy</artifactId> - </exclusion> - <exclusion> - <groupId>commons-digester</groupId> - <artifactId>commons-digester</artifactId> - </exclusion> - <exclusion> - <groupId>com.codahale.metrics</groupId> - <artifactId>metrics-core</artifactId> - </exclusion> - <exclusion> - <groupId>com.codahale.metrics</groupId> - <artifactId>metrics-servlets</artifactId> - </exclusion> - <exclusion> - <groupId>javax.xml.bind</groupId> - <artifactId>jaxb-api</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.inject</groupId> - <artifactId>guice</artifactId> - </exclusion> - <exclusion> - <groupId>com.sun.jersey.contribs</groupId> - <artifactId>jersey-guice</artifactId> - </exclusion> - <exclusion> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jetty-sslengine</artifactId> - </exclusion> - <exclusion> - <groupId>org.codehaus.jettison</groupId> - <artifactId>jettison</artifactId> - </exclusion> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> <groupId>org.codehaus.jettison</groupId> <artifactId>jettison</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/hive/blob/11ddb8be/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java index c906a5d..2445075 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java @@ -52,8 +52,8 @@ public class LlapOptionsProcessor { public static final String OPTION_AUXHBASE = "auxhbase"; // used to localize jars public static final String OPTION_JAVA_HOME = "javaHome"; // forward via config.json public static final String OPTION_HIVECONF = "hiveconf"; // llap-daemon-site if relevant parameter - public static final String OPTION_SLIDER_AM_CONTAINER_MB = "slider-am-container-mb"; // forward as arg - public static final String OPTION_SLIDER_APPCONFIG_GLOBAL = "slider-appconfig-global"; // forward as arg + public static final String OPTION_SERVICE_AM_CONTAINER_MB = "service-am-container-mb"; // forward as arg + public static final String OPTION_SERVICE_APPCONFIG_GLOBAL = "service-appconfig-global"; // forward as arg public static final String OPTION_LLAP_QUEUE = "queue"; // forward via config.json public static final String OPTION_IO_THREADS = "iothreads"; // llap-daemon-site @@ -61,12 +61,11 @@ public class LlapOptionsProcessor { public static final String OPTION_ARGS = "args"; // forward as arg public static final String OPTION_LOGLEVEL = "loglevel"; // forward as arg public static final String OPTION_LOGGER = "logger"; // forward as arg - public static final String OPTION_CHAOS_MONKEY = "chaosmonkey"; // forward as arg - public static final String OPTION_SLIDER_KEYTAB_DIR = "slider-keytab-dir"; - public static final String OPTION_SLIDER_KEYTAB = "slider-keytab"; - public static final String OPTION_SLIDER_PRINCIPAL = "slider-principal"; - public static final String OPTION_SLIDER_PLACEMENT = "slider-placement"; - public static final String OPTION_SLIDER_DEFAULT_KEYTAB = "slider-default-keytab"; + public static final String OPTION_SERVICE_KEYTAB_DIR = "service-keytab-dir"; + public static final String OPTION_SERVICE_KEYTAB = "service-keytab"; + public static final String OPTION_SERVICE_PRINCIPAL = "service-principal"; + public static final String OPTION_SERVICE_PLACEMENT = "service-placement"; + public static final String OPTION_SERVICE_DEFAULT_KEYTAB = "service-default-keytab"; public static final String OPTION_OUTPUT_DIR = "output"; public static final String OPTION_START = "startImmediately"; public static final String OPTION_HEALTH_PERCENT = "health-percent"; @@ -217,24 +216,21 @@ public class LlapOptionsProcessor { LogHelpers.LLAP_LOGGER_NAME_QUERY_ROUTING + ", " + LogHelpers.LLAP_LOGGER_NAME_CONSOLE) .create()); - options.addOption(OptionBuilder.hasArg().withArgName(OPTION_CHAOS_MONKEY).withLongOpt(OPTION_CHAOS_MONKEY) - .withDescription("chaosmonkey interval").create('m')); + options.addOption(OptionBuilder.hasArg(false).withArgName(OPTION_SERVICE_DEFAULT_KEYTAB).withLongOpt(OPTION_SERVICE_DEFAULT_KEYTAB) + .withDescription("try to set default settings for Service AM keytab; mostly for dev testing").create()); - options.addOption(OptionBuilder.hasArg(false).withArgName(OPTION_SLIDER_DEFAULT_KEYTAB).withLongOpt(OPTION_SLIDER_DEFAULT_KEYTAB) - .withDescription("try to set default settings for Slider AM keytab; mostly for dev testing").create()); + options.addOption(OptionBuilder.hasArg().withArgName(OPTION_SERVICE_KEYTAB_DIR).withLongOpt(OPTION_SERVICE_KEYTAB_DIR) + .withDescription("Service AM keytab directory on HDFS (where the headless user keytab is stored by Service keytab installation, e.g. .yarn/keytabs/llap)").create()); - options.addOption(OptionBuilder.hasArg().withArgName(OPTION_SLIDER_KEYTAB_DIR).withLongOpt(OPTION_SLIDER_KEYTAB_DIR) - .withDescription("Slider AM keytab directory on HDFS (where the headless user keytab is stored by Slider keytab installation, e.g. .slider/keytabs/llap)").create()); + options.addOption(OptionBuilder.hasArg().withArgName(OPTION_SERVICE_KEYTAB).withLongOpt(OPTION_SERVICE_KEYTAB) + .withDescription("Service AM keytab file name inside " + OPTION_SERVICE_KEYTAB_DIR).create()); - options.addOption(OptionBuilder.hasArg().withArgName(OPTION_SLIDER_KEYTAB).withLongOpt(OPTION_SLIDER_KEYTAB) - .withDescription("Slider AM keytab file name inside " + OPTION_SLIDER_KEYTAB_DIR).create()); + options.addOption(OptionBuilder.hasArg().withArgName(OPTION_SERVICE_PRINCIPAL).withLongOpt(OPTION_SERVICE_PRINCIPAL) + .withDescription("Service AM principal; should be the user running the cluster, e.g. h...@example.com").create()); - options.addOption(OptionBuilder.hasArg().withArgName(OPTION_SLIDER_PRINCIPAL).withLongOpt(OPTION_SLIDER_PRINCIPAL) - .withDescription("Slider AM principal; should be the user running the cluster, e.g. h...@example.com").create()); - - options.addOption(OptionBuilder.hasArg().withArgName(OPTION_SLIDER_PLACEMENT).withLongOpt(OPTION_SLIDER_PLACEMENT) - .withDescription("Slider placement policy; see slider documentation at https://slider.incubator.apache.org/docs/placement.html." - + " 4 means anti-affinity (the default; unnecessary if LLAP is going to take more than half of the YARN capacity of a node), 0 is normal.").create()); + options.addOption(OptionBuilder.hasArg().withArgName(OPTION_SERVICE_PLACEMENT).withLongOpt(OPTION_SERVICE_PLACEMENT) + .withDescription("Service placement policy; see YARN documentation at https://issues.apache.org/jira/browse/YARN-1042." + + " This is unnecessary if LLAP is going to take more than half of the YARN capacity of a node.").create()); options.addOption(OptionBuilder.hasArg().withArgName(OPTION_HEALTH_PERCENT).withLongOpt(OPTION_HEALTH_PERCENT) .withDescription("Percentage of running containers after which LLAP application is considered healthy" + @@ -291,12 +287,12 @@ public class LlapOptionsProcessor { .create()); options.addOption(OptionBuilder.hasArg().withArgName("b") - .withLongOpt(OPTION_SLIDER_AM_CONTAINER_MB) - .withDescription("The size of the slider AppMaster container in MB").create('b')); + .withLongOpt(OPTION_SERVICE_AM_CONTAINER_MB) + .withDescription("The size of the service AppMaster container in MB").create('b')); options.addOption(OptionBuilder.withValueSeparator().hasArgs(2).withArgName("property=value") - .withLongOpt(OPTION_SLIDER_APPCONFIG_GLOBAL) - .withDescription("Property (key=value) to be set in the global section of the Slider appConfig") + .withLongOpt(OPTION_SERVICE_APPCONFIG_GLOBAL) + .withDescription("Property (key=value) to be set in the global section of the Service appConfig") .create()); options.addOption(OptionBuilder.hasArg().withArgName(OPTION_IO_THREADS) @@ -366,7 +362,7 @@ public class LlapOptionsProcessor { logger.equalsIgnoreCase(LogHelpers.LLAP_LOGGER_NAME_RFA)); } - // loglevel, chaosmonkey & args are parsed by the python processor + // loglevel & args are parsed by the python processor return new LlapOptions(name, instances, directory, executors, ioThreads, cache, size, xmx, jars, isHbase, hiveconf, javaHome, queueName, logger, doStart, output, isHiveAux); http://git-wip-us.apache.org/repos/asf/hive/blob/11ddb8be/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java index b9845d4..20fec7e 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java @@ -95,7 +95,7 @@ public class LlapServiceDriver { private static final String HBASE_SERDE_CLASS = "org.apache.hadoop.hive.hbase.HBaseSerDe"; private static final String[] NEEDED_CONFIGS = LlapDaemonConfiguration.DAEMON_CONFIGS; private static final String[] OPTIONAL_CONFIGS = LlapDaemonConfiguration.SSL_DAEMON_CONFIGS; - private static final String OUTPUT_DIR_PREFIX = "llap-slider-"; + private static final String OUTPUT_DIR_PREFIX = "llap-yarn-"; // This is not a config that users set in hive-site. It's only use is to share information // between the java component of the service driver and the python component. @@ -596,8 +596,9 @@ public class LlapServiceDriver { } rc = runPackagePy(args, tmpDir, scriptParent, version, outputDir); if (rc == 0) { - LlapSliderUtils.startCluster(conf, options.getName(), "llap-" + version + ".zip", - packageDir, HiveConf.getVar(conf, ConfVars.LLAP_DAEMON_QUEUE_NAME)); + LlapSliderUtils.startCluster(conf, options.getName(), + "llap-" + version + ".tar.gz", packageDir, + HiveConf.getVar(conf, ConfVars.LLAP_DAEMON_QUEUE_NAME)); } } else { rc = 0; @@ -620,7 +621,7 @@ public class LlapServiceDriver { private int runPackagePy(String[] args, Path tmpDir, Path scriptParent, String version, String outputDir) throws IOException, InterruptedException { - Path scriptPath = new Path(new Path(scriptParent, "slider"), "package.py"); + Path scriptPath = new Path(new Path(scriptParent, "yarn"), "package.py"); List<String> scriptArgs = new ArrayList<>(args.length + 7); scriptArgs.add("python"); scriptArgs.add(scriptPath.toString()); @@ -651,7 +652,7 @@ public class LlapServiceDriver { private JSONObject createConfigJson(long containerSize, long cache, long xmx, String java_home) throws JSONException { - // extract configs for processing by the python fragments in Slider + // extract configs for processing by the python fragments in YARN Service JSONObject configs = new JSONObject(); configs.put("java.home", java_home); http://git-wip-us.apache.org/repos/asf/hive/blob/11ddb8be/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapSliderUtils.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapSliderUtils.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapSliderUtils.java index 8e5ae09..af47b26 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapSliderUtils.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapSliderUtils.java @@ -18,60 +18,53 @@ package org.apache.hadoop.hive.llap.cli; -import com.google.common.io.Files; import java.io.File; import java.io.IOException; + import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.llap.cli.status.LlapStatusHelpers.AppStatusBuilder; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.client.ServiceClient; +import org.apache.hadoop.yarn.service.utils.CoreFileSystem; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.SystemClock; -import org.apache.slider.api.types.ApplicationDiagnostics; -import org.apache.slider.client.SliderClient; -import org.apache.slider.common.params.ActionCreateArgs; -import org.apache.slider.common.params.ActionDestroyArgs; -import org.apache.slider.common.params.ActionFreezeArgs; -import org.apache.slider.common.params.ActionInstallPackageArgs; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.exceptions.UnknownApplicationInstanceException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class LlapSliderUtils { - private static final String SLIDER_GZ = "slider-agent.tar.gz"; - private static final Logger LOG = LoggerFactory.getLogger(LlapSliderUtils.class); + private static final Logger LOG = LoggerFactory + .getLogger(LlapSliderUtils.class); + private static final String LLAP_PACKAGE_DIR = ".yarn/package/LLAP/"; - public static SliderClient createSliderClient( + public static ServiceClient createServiceClient( Configuration conf) throws Exception { - SliderClient sliderClient = new SliderClient() { - @Override - public void serviceInit(Configuration conf) throws Exception { - super.serviceInit(conf); - initHadoopBinding(); - } - }; - Configuration sliderClientConf = new Configuration(conf); - sliderClientConf = sliderClient.bindArgs(sliderClientConf, - new String[]{"help"}); - sliderClient.init(sliderClientConf); - sliderClient.start(); - return sliderClient; + ServiceClient serviceClient = new ServiceClient(); + serviceClient.init(conf); + serviceClient.start(); + return serviceClient; } - public static ApplicationReport getAppReport(String appName, SliderClient sliderClient, + public static ApplicationReport getAppReport(String appName, ServiceClient serviceClient, long timeoutMs) throws LlapStatusServiceDriver.LlapStatusCliException { - Clock clock = new SystemClock(); + Clock clock = SystemClock.getInstance(); long startTime = clock.getTime(); long timeoutTime = timeoutMs < 0 ? Long.MAX_VALUE : (startTime + timeoutMs); ApplicationReport appReport = null; + ApplicationId appId; + try { + appId = serviceClient.getAppId(appName); + } catch (YarnException | IOException e) { + return null; + } while (appReport == null) { try { - appReport = sliderClient.getYarnAppListClient().findInstance(appName); + appReport = serviceClient.getYarnClient().getApplicationReport(appId); if (timeoutMs == 0) { // break immediately if timeout is 0 break; @@ -94,150 +87,78 @@ public class LlapSliderUtils { return appReport; } - public static ApplicationDiagnostics getApplicationDiagnosticsFromYarnDiagnostics( - ApplicationReport appReport, AppStatusBuilder appStatusBuilder, Logger LOG) { - if (appReport == null) { - return null; - } - String diagnostics = appReport.getDiagnostics(); - if (diagnostics == null || diagnostics.isEmpty()) { - return null; + public static Service getService(Configuration conf, String name) { + LOG.info("Get service details for " + name); + ServiceClient sc; + try { + sc = createServiceClient(conf); + } catch (Exception e) { + throw new RuntimeException(e); } + Service service = null; try { - return ApplicationDiagnostics.fromJson(diagnostics); - } catch (IOException e) { - LOG.warn("Failed to parse application diagnostics from Yarn Diagnostics - {}", diagnostics); - // Set the raw YARN diagnostics here - the caller won't know if they even exist. - appStatusBuilder.setDiagnostics(diagnostics); - return null; + service = sc.getStatus(name); + } catch (YarnException | IOException e) { + // Probably the app does not exist + LOG.info(e.getLocalizedMessage()); + throw new RuntimeException(e); + } finally { + try { + sc.close(); + } catch (IOException e) { + LOG.info("Failed to close service client", e); + } } + return service; } public static void startCluster(Configuration conf, String name, String packageName, Path packageDir, String queue) { LOG.info("Starting cluster with " + name + ", " + packageName + ", " + queue + ", " + packageDir); - SliderClient sc; + ServiceClient sc; try { - sc = createSliderClient(conf); + sc = createServiceClient(conf); } catch (Exception e) { throw new RuntimeException(e); } try { - LOG.info("Executing the freeze command"); - ActionFreezeArgs freezeArgs = new ActionFreezeArgs(); - freezeArgs.force = true; - freezeArgs.setWaittime(3600); // Wait forever (or at least for an hour). try { - sc.actionFreeze(name, freezeArgs); - } catch (UnknownApplicationInstanceException ex) { - LOG.info("There was no old application instance to freeze"); + LOG.info("Executing the stop command"); + sc.actionStop(name, true); + } catch (Exception ex) { + // Ignore exceptions from stop + LOG.info(ex.getLocalizedMessage()); } - LOG.info("Executing the destroy command"); - ActionDestroyArgs destroyArg = new ActionDestroyArgs(); - destroyArg.force = true; try { - sc.actionDestroy(name, destroyArg); - } catch (UnknownApplicationInstanceException ex) { - LOG.info("There was no old application instance to destroy"); - } - LOG.info("Executing the install command"); - ActionInstallPackageArgs installArgs = new ActionInstallPackageArgs(); - installArgs.name = "LLAP"; - installArgs.packageURI = new Path(packageDir, packageName).toString(); - installArgs.replacePkg = true; - sc.actionInstallPkg(installArgs); - LOG.info("Executing the create command"); - ActionCreateArgs createArgs = new ActionCreateArgs(); - createArgs.resources = new File(new Path(packageDir, "resources.json").toString()); - createArgs.template = new File(new Path(packageDir, "appConfig.json").toString()); - createArgs.setWaittime(3600); - if (queue != null) { - createArgs.queue = queue; + LOG.info("Executing the destroy command"); + sc.actionDestroy(name); + } catch (Exception ex) { + // Ignore exceptions from destroy + LOG.info(ex.getLocalizedMessage()); } - // See the comments in the method. SliderClient doesn't work in normal circumstances. - File bogusSliderFile = startSetSliderLibDir(); + LOG.info("Uploading the app tarball"); + CoreFileSystem fs = new CoreFileSystem(conf); + fs.copyLocalFileToHdfs(new File(packageDir.toString(), packageName), + new Path(LLAP_PACKAGE_DIR), new FsPermission("755")); + + LOG.info("Executing the launch command"); + File yarnfile = new File(new Path(packageDir, "Yarnfile").toString()); + Long lifetime = null; // unlimited lifetime try { - sc.actionCreate(name, createArgs); + sc.actionLaunch(yarnfile.getAbsolutePath(), name, lifetime, queue); } finally { - endSetSliderLibDir(bogusSliderFile); } - LOG.debug("Started the cluster via slider API"); + LOG.debug("Started the cluster via service API"); } catch (YarnException | IOException e) { throw new RuntimeException(e); } finally { try { sc.close(); } catch (IOException e) { - LOG.info("Failed to close slider client", e); - } - } - } - - public static File startSetSliderLibDir() throws IOException { - // TODO: this is currently required for the use of slider create API. Need SLIDER-1192. - File sliderJarDir = SliderUtils.findContainingJar(SliderClient.class).getParentFile(); - File gz = new File(sliderJarDir, SLIDER_GZ); - if (gz.exists()) { - String path = sliderJarDir.getAbsolutePath(); - LOG.info("Setting slider.libdir based on jar file location: " + path); - System.setProperty("slider.libdir", path); - return null; - } - - // There's no gz file next to slider jars. Due to the horror that is SliderClient, we'd have - // to find it and copy it there. Let's try to find it. Also set slider.libdir. - String path = System.getProperty("slider.libdir"); - gz = null; - if (path != null && !path.isEmpty()) { - LOG.info("slider.libdir was already set: " + path); - gz = new File(path, SLIDER_GZ); - if (!gz.exists()) { - gz = null; + LOG.info("Failed to close service client", e); } } - if (gz == null) { - path = System.getenv("SLIDER_HOME"); - if (path != null && !path.isEmpty()) { - gz = new File(new File(path, "lib"), SLIDER_GZ); - if (gz.exists()) { - path = gz.getParentFile().getAbsolutePath(); - LOG.info("Setting slider.libdir based on SLIDER_HOME: " + path); - System.setProperty("slider.libdir", path); - } else { - gz = null; - } - } - } - if (gz == null) { - // This is a terrible hack trying to find slider on a typical installation. Sigh... - File rootDir = SliderUtils.findContainingJar(HiveConf.class) - .getParentFile().getParentFile().getParentFile(); - File sliderJarDir2 = new File(new File(rootDir, "slider"), "lib"); - if (sliderJarDir2.exists()) { - gz = new File(sliderJarDir2, SLIDER_GZ); - if (gz.exists()) { - path = sliderJarDir2.getAbsolutePath(); - LOG.info("Setting slider.libdir based on guesswork: " + path); - System.setProperty("slider.libdir", path); - } else { - gz = null; - } - } - } - if (gz == null) { - throw new IOException("Cannot find " + SLIDER_GZ + ". Please ensure SLIDER_HOME is set."); - } - File newGz = new File(sliderJarDir, SLIDER_GZ); - LOG.info("Copying " + gz + " to " + newGz); - Files.copy(gz, newGz); - newGz.deleteOnExit(); - return newGz; } - public static void endSetSliderLibDir(File newGz) throws IOException { - if (newGz == null || !newGz.exists()) return; - LOG.info("Deleting " + newGz); - newGz.delete(); - } } http://git-wip-us.apache.org/repos/asf/hive/blob/11ddb8be/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java index 65b4d81..e0ada45 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java @@ -25,7 +25,6 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; import java.io.PrintWriter; -import java.net.URISyntaxException; import java.text.DecimalFormat; import java.util.Arrays; import java.util.Collection; @@ -52,17 +51,12 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.service.api.records.Container; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.api.records.ServiceState; +import org.apache.hadoop.yarn.service.client.ServiceClient; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.SystemClock; -import org.apache.slider.api.ClusterDescription; -import org.apache.slider.api.ClusterDescriptionKeys; -import org.apache.slider.api.StateValues; -import org.apache.slider.api.StatusKeys; -import org.apache.slider.api.types.ApplicationDiagnostics; -import org.apache.slider.api.types.ContainerInformation; -import org.apache.slider.client.SliderClient; -import org.apache.slider.common.params.ActionDiagnosticArgs; -import org.apache.slider.core.exceptions.SliderException; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.map.SerializationConfig; import org.codehaus.jackson.map.annotate.JsonSerialize; @@ -71,7 +65,7 @@ import org.slf4j.LoggerFactory; public class LlapStatusServiceDriver { - private static final EnumSet<State> NO_SLIDER_INFO_STATES = EnumSet.of( + private static final EnumSet<State> NO_YARN_SERVICE_INFO_STATES = EnumSet.of( State.APP_NOT_FOUND, State.COMPLETE, State.LAUNCHING); private static final EnumSet<State> LAUNCHING_STATES = EnumSet.of( State.LAUNCHING, State.RUNNING_PARTIAL, State.RUNNING_ALL); @@ -119,11 +113,11 @@ public class LlapStatusServiceDriver { private static final long LOG_SUMMARY_INTERVAL = 15000L; // Log summary every ~15 seconds. - private static final String LLAP_KEY = "LLAP"; + private static final String LLAP_KEY = "llap"; private final Configuration conf; private final Clock clock = new SystemClock(); private String appName = null; - private SliderClient sliderClient = null; + private ServiceClient serviceClient = null; private Configuration llapRegistryConf = null; private LlapRegistryService llapRegistry = null; @@ -203,7 +197,7 @@ public class LlapStatusServiceDriver { if (StringUtils.isEmpty(appName)) { appName = HiveConf.getVar(conf, HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS); if (appName.startsWith("@") && appName.length() > 1) { - // This is a valid slider app name. Parse it out. + // This is a valid YARN Service name. Parse it out. appName = appName.substring(1); } else { // Invalid app name. Checked later. @@ -213,7 +207,7 @@ public class LlapStatusServiceDriver { if (StringUtils.isEmpty(appName)) { String message = "Invalid app name. This must be setup via config or passed in as a parameter." + - " This tool works with clusters deployed by Slider/YARN"; + " This tool works with clusters deployed by YARN Service"; LOG.info(message); return ExitCode.INCORRECT_USAGE.getInt(); } @@ -225,13 +219,13 @@ public class LlapStatusServiceDriver { } try { - if (sliderClient == null) { - sliderClient = LlapSliderUtils.createSliderClient(conf); + if (serviceClient == null) { + serviceClient = LlapSliderUtils.createServiceClient(conf); } } catch (Exception e) { LlapStatusCliException le = new LlapStatusCliException( - LlapStatusServiceDriver.ExitCode.SLIDER_CLIENT_ERROR_CREATE_FAILED, - "Failed to create slider client", e); + LlapStatusServiceDriver.ExitCode.SERVICE_CLIENT_ERROR_CREATE_FAILED, + "Failed to create service client", e); logError(le); return le.getExitCode().getInt(); } @@ -239,13 +233,14 @@ public class LlapStatusServiceDriver { // Get the App report from YARN ApplicationReport appReport; try { - appReport = LlapSliderUtils.getAppReport(appName, sliderClient, options.getFindAppTimeoutMs()); + appReport = LlapSliderUtils.getAppReport(appName, serviceClient, + options.getFindAppTimeoutMs()); } catch (LlapStatusCliException e) { logError(e); return e.getExitCode().getInt(); } - // Process the report to decide whether to go to slider. + // Process the report ExitCode ret; try { ret = processAppReport(appReport, appStatusBuilder); @@ -256,26 +251,16 @@ public class LlapStatusServiceDriver { if (ret != ExitCode.SUCCESS) { return ret.getInt(); - } else if (NO_SLIDER_INFO_STATES.contains(appStatusBuilder.getState())) { + } else if (NO_YARN_SERVICE_INFO_STATES.contains(appStatusBuilder.getState())) { return ExitCode.SUCCESS.getInt(); } else { - // Get information from slider. + // Get information from YARN Service try { - ret = populateAppStatusFromSliderStatus(appName, sliderClient, appStatusBuilder); - } catch (LlapStatusCliException e) { - // In case of failure, send back whatever is constructed sop far - which wouldbe from the AppReport - logError(e); - return e.getExitCode().getInt(); - } - } - - - if (ret != ExitCode.SUCCESS) { - return ret.getInt(); - } else { - try { - ret = populateAppStatusFromSliderDiagnostics(appName, sliderClient, appStatusBuilder); + ret = populateAppStatusFromServiceStatus(appName, serviceClient, + appStatusBuilder); } catch (LlapStatusCliException e) { + // In case of failure, send back whatever is constructed so far - + // which would be from the AppReport logError(e); return e.getExitCode().getInt(); } @@ -315,57 +300,6 @@ public class LlapStatusServiceDriver { } } - private SliderClient createSliderClient() throws LlapStatusCliException { - if (sliderClient != null) { - return sliderClient; - } - - try { - sliderClient = LlapSliderUtils.createSliderClient(conf); - } catch (Exception e) { - throw new LlapStatusCliException(ExitCode.SLIDER_CLIENT_ERROR_CREATE_FAILED, - "Failed to create slider client", e); - } - return sliderClient; - } - - private ApplicationReport getAppReport(String appName, SliderClient sliderClient, - long timeoutMs) throws LlapStatusCliException { - - long startTime = clock.getTime(); - long timeoutTime = timeoutMs < 0 ? Long.MAX_VALUE : (startTime + timeoutMs); - ApplicationReport appReport = null; - - // TODO HIVE-13454 Maybe add an option to wait for a certain amount of time for the app to - // move to running state. Potentially even wait for the containers to be launched. - -// while (clock.getTime() < timeoutTime && appReport == null) { - - while (appReport == null) { - try { - appReport = sliderClient.getYarnAppListClient().findInstance(appName); - if (timeoutMs == 0) { - // break immediately if timeout is 0 - break; - } - // Otherwise sleep, and try again. - if (appReport == null) { - long remainingTime = Math.min(timeoutTime - clock.getTime(), 500l); - if (remainingTime > 0) { - Thread.sleep(remainingTime); - } else { - break; - } - } - } catch (Exception e) { // No point separating IOException vs YarnException vs others - throw new LlapStatusCliException(ExitCode.YARN_ERROR, - "Failed to get Yarn AppReport", e); - } - } - return appReport; - } - - /** * Populates parts of the AppStatus * @@ -399,7 +333,7 @@ public class LlapStatusServiceDriver { return ExitCode.SUCCESS; case RUNNING: appStatusBuilder.maybeCreateAndGetAmInfo().setAppId(appReport.getApplicationId().toString()); - // If the app state is running, get additional information from Slider itself. + // If the app state is running, get additional information from YARN Service return ExitCode.SUCCESS; case FINISHED: case FAILED: @@ -407,14 +341,7 @@ public class LlapStatusServiceDriver { appStatusBuilder.maybeCreateAndGetAmInfo().setAppId(appReport.getApplicationId().toString()); appStatusBuilder.setAppFinishTime(appReport.getFinishTime()); appStatusBuilder.setState(State.COMPLETE); - ApplicationDiagnostics appDiagnostics = - LlapSliderUtils.getApplicationDiagnosticsFromYarnDiagnostics( - appReport, appStatusBuilder, LOG); - if (appDiagnostics == null) { - LOG.warn("AppDiagnostics not available for YARN application report"); - } else { - processAppDiagnostics(appStatusBuilder, appDiagnostics, true); - } + // add log links and other diagnostics from YARN Service return ExitCode.SUCCESS; default: throw new LlapStatusCliException(ExitCode.INTERNAL_ERROR, @@ -422,135 +349,57 @@ public class LlapStatusServiceDriver { } } - - - - /** - * Populates information from SliderStatus. + * Populates information from YARN Service Status. * * @param appName - * @param sliderClient + * @param serviceClient * @param appStatusBuilder - * @return an ExitCode. An ExitCode other than ExitCode.SUCCESS implies future progress not possible + * @return an ExitCode. An ExitCode other than ExitCode.SUCCESS implies future + * progress not possible * @throws LlapStatusCliException */ - private ExitCode populateAppStatusFromSliderStatus(String appName, SliderClient sliderClient, AppStatusBuilder appStatusBuilder) throws - LlapStatusCliException { - - ClusterDescription clusterDescription; + private ExitCode populateAppStatusFromServiceStatus(String appName, + ServiceClient serviceClient, AppStatusBuilder appStatusBuilder) + throws LlapStatusCliException { + ExitCode exitCode = ExitCode.YARN_ERROR; try { - clusterDescription = sliderClient.getClusterDescription(appName); - } catch (SliderException e) { - throw new LlapStatusCliException(ExitCode.SLIDER_CLIENT_ERROR_OTHER, - "Failed to get cluster description from slider. SliderErrorCode=" + (e).getExitCode(), e); - } catch (Exception e) { - throw new LlapStatusCliException(ExitCode.SLIDER_CLIENT_ERROR_OTHER, - "Failed to get cluster description from slider", e); - } - - if (clusterDescription == null) { - LOG.info("Slider ClusterDescription not available"); - return ExitCode.SLIDER_CLIENT_ERROR_OTHER; // ClusterDescription should always be present. - } else { - // Process the Cluster Status returned by slider. - appStatusBuilder.setOriginalConfigurationPath(clusterDescription.originConfigurationPath); - appStatusBuilder.setGeneratedConfigurationPath(clusterDescription.generatedConfigurationPath); - appStatusBuilder.setAppStartTime(clusterDescription.createTime); - - // Finish populating AMInfo - appStatusBuilder.maybeCreateAndGetAmInfo().setAmWebUrl(clusterDescription.getInfo(StatusKeys.INFO_AM_WEB_URL)); - appStatusBuilder.maybeCreateAndGetAmInfo().setHostname(clusterDescription.getInfo(StatusKeys.INFO_AM_HOSTNAME)); - appStatusBuilder.maybeCreateAndGetAmInfo().setContainerId(clusterDescription.getInfo(StatusKeys.INFO_AM_CONTAINER_ID)); - - - if (clusterDescription.statistics != null) { - Map<String, Integer> llapStats = clusterDescription.statistics.get(LLAP_KEY); - if (llapStats != null) { - int desiredContainers = llapStats.get(StatusKeys.STATISTICS_CONTAINERS_DESIRED); - int liveContainers = llapStats.get(StatusKeys.STATISTICS_CONTAINERS_LIVE); - appStatusBuilder.setDesiredInstances(desiredContainers); - appStatusBuilder.setLiveInstances(liveContainers); - } else { - throw new LlapStatusCliException(ExitCode.SLIDER_CLIENT_ERROR_OTHER, - "Failed to get statistics for LLAP"); // Error since LLAP should always exist. + Service service = serviceClient.getStatus(appName); + if (service != null) { + // How to get config paths and AmInfo + ServiceState state = service.getState(); + appStatusBuilder.setAppStartTime(service.getLaunchTime() == null ? 0 + : service.getLaunchTime().getTime()); + appStatusBuilder.setDesiredInstances( + service.getComponent(LLAP_KEY).getNumberOfContainers() == null ? 0 + : service.getComponent(LLAP_KEY).getNumberOfContainers() + .intValue()); + appStatusBuilder.setLiveInstances( + service.getComponent(LLAP_KEY).getContainers().size()); + for (Container cont : service.getComponent(LLAP_KEY).getContainers()) { + LlapInstance llapInstance = new LlapInstance(cont.getHostname(), + cont.getId()); + appStatusBuilder.addNewRunningLlapInstance(llapInstance); } - // TODO HIVE-13454 Use some information from here such as containers.start.failed - // and containers.failed.recently to provide an estimate of whether this app is healthy or not. - } else { - throw new LlapStatusCliException(ExitCode.SLIDER_CLIENT_ERROR_OTHER, - "Failed to get statistics"); // Error since statistics should always exist. - } - - // Code to locate container status via slider. Not using this at the moment. - if (clusterDescription.status != null) { - Object liveObject = clusterDescription.status.get(ClusterDescriptionKeys.KEY_CLUSTER_LIVE); - if (liveObject != null) { - Map<String, Map<String, Map<String, Object>>> liveEntity = - (Map<String, Map<String, Map<String, Object>>>) liveObject; - Map<String, Map<String, Object>> llapEntity = liveEntity.get(LLAP_KEY); - - if (llapEntity != null) { // Not a problem. Nothing has come up yet. - for (Map.Entry<String, Map<String, Object>> containerEntry : llapEntity.entrySet()) { - String containerIdString = containerEntry.getKey(); - Map<String, Object> containerParams = containerEntry.getValue(); - - String host = (String) containerParams.get("host"); - - LlapInstance - llapInstance = new LlapInstance(host, containerIdString); - - appStatusBuilder.addNewRunningLlapInstance(llapInstance); - } - } - + if (state == ServiceState.STABLE) { + exitCode = ExitCode.SUCCESS; } + } else { + exitCode = ExitCode.SERVICE_CLIENT_ERROR_OTHER; } - - return ExitCode.SUCCESS; - - } - } - - /** - * Populates information based on the slider diagnostics call. Must be invoked - * after populating status from slider status. - * @param appName - * @param sliderClient - * @param appStatusBuilder - * @return - * @throws LlapStatusCliException - */ - private ExitCode populateAppStatusFromSliderDiagnostics(String appName, - SliderClient sliderClient, - AppStatusBuilder appStatusBuilder) throws - LlapStatusCliException { - - ApplicationDiagnostics appDiagnostics; - try { - ActionDiagnosticArgs args = new ActionDiagnosticArgs(); - args.containers = true; - args.name = appName; - appDiagnostics = - sliderClient.actionDiagnosticContainers(args); - } catch (YarnException | IOException | URISyntaxException e) { - throw new LlapStatusCliException( - ExitCode.SLIDER_CLIENT_ERROR_OTHER, - "Failed to get container diagnostics from slider", e); - } - if (appDiagnostics == null) { - LOG.info("Slider container diagnostics not available"); - return ExitCode.SLIDER_CLIENT_ERROR_OTHER; - } - - processAppDiagnostics(appStatusBuilder, appDiagnostics, false); - - return ExitCode.SUCCESS; + } catch (IOException | YarnException e) { + LlapStatusCliException le = new LlapStatusCliException( + LlapStatusServiceDriver.ExitCode.SERVICE_CLIENT_ERROR_OTHER, + "Failed to get service status", e); + logError(le); + exitCode = le.getExitCode(); + } + return exitCode; } /** * Populate additional information for containers from the LLAP registry. Must be invoked - * after Slider status. Also after slider-diagnostics. + * after YARN Service status and diagnostics. * @param appStatusBuilder * @return an ExitCode. An ExitCode other than ExitCode.SUCCESS implies future progress not possible * @throws LlapStatusCliException @@ -584,7 +433,7 @@ public class LlapStatusServiceDriver { appStatusBuilder.clearRunningLlapInstances(); return ExitCode.SUCCESS; } else { - // Tracks instances known by both slider and llap. + // Tracks instances known by both YARN Service and llap. List<LlapInstance> validatedInstances = new LinkedList<>(); List<String> llapExtraInstances = new LinkedList<>(); @@ -612,7 +461,8 @@ public class LlapStatusServiceDriver { appStatusBuilder.setLiveInstances(validatedInstances.size()); appStatusBuilder.setLaunchingInstances(llapExtraInstances.size()); - if (validatedInstances.size() >= appStatusBuilder.getDesiredInstances()) { + if (appStatusBuilder.getDesiredInstances() != null && validatedInstances + .size() >= appStatusBuilder.getDesiredInstances()) { appStatusBuilder.setState(State.RUNNING_ALL); if (validatedInstances.size() > appStatusBuilder.getDesiredInstances()) { LOG.warn("Found more entries in LLAP registry, as compared to desired entries"); @@ -633,7 +483,7 @@ public class LlapStatusServiceDriver { } if (llapExtraInstances.size() > 0) { // Old containers which are likely shutting down, or new containers which - // launched between slider-status/slider-diagnostics. Skip for this iteration. + // launched between YARN Service status/diagnostics. Skip for this iteration. LOG.debug("Instances likely to shutdown soon: {}", llapExtraInstances); } @@ -643,52 +493,6 @@ public class LlapStatusServiceDriver { return ExitCode.SUCCESS; } - - private static void processAppDiagnostics(AppStatusBuilder appStatusBuilder, - ApplicationDiagnostics appDiagnostics, boolean appComplete) { - // For a running app this should be empty. - String finalMessage = appDiagnostics.getFinalMessage(); - Collection<ContainerInformation> containerInfos = - appDiagnostics.getContainers(); - appStatusBuilder.setDiagnostics(finalMessage); - if (containerInfos != null) { - for (ContainerInformation containerInformation : containerInfos) { - if (containerInformation.getState() == StateValues.STATE_LIVE && !appComplete) { - LlapInstance instance = appStatusBuilder - .removeAndGetCompletedLlapInstanceForContainer( - containerInformation.getContainerId()); - if (instance == - null) { // New launch. Not available during slider status, but available now. - instance = new LlapInstance(containerInformation.getHost(), - containerInformation.getContainerId()); - } - instance.setLogUrl(containerInformation.getLogLink()); - appStatusBuilder.addNewRunningLlapInstance(instance); - } else if (containerInformation.getState() == - StateValues.STATE_STOPPED || appComplete) { - LlapInstance instance = - new LlapInstance(containerInformation.getHost(), - containerInformation.getContainerId()); - instance.setLogUrl(containerInformation.getLogLink()); - if (appComplete && containerInformation.getExitCode() != - ContainerExitStatus.INVALID) { - instance - .setYarnContainerExitStatus(containerInformation.getExitCode()); - } - instance.setDiagnostics(containerInformation.getDiagnostics()); - appStatusBuilder.addNewCompleteLlapInstance(instance); - } else { - LOG.warn("Unexpected containerstate={}, for container={}", - containerInformation.getState(), containerInformation); - } - } - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("ContainerInfos is null"); - } - } - } - private static String constructCompletedContainerDiagnostics(List<LlapInstance> completedInstances) { StringBuilder sb = new StringBuilder(); if (completedInstances == null || completedInstances.size() == 0) { @@ -709,7 +513,7 @@ public class LlapStatusServiceDriver { ContainerExitStatus.KILLED_EXCEEDED_VMEM) { sb.append("\tKILLED container (by YARN for exceeding memory limits): "); } else { - // TODO HIVE-15865 Handle additional reasons like OS launch failed (Slider needs to give this info) + // TODO HIVE-15865 Handle additional reasons like OS launch failed sb.append("\tFAILED container: "); } sb.append(" ").append(instance.getContainerId()); @@ -763,8 +567,8 @@ public class LlapStatusServiceDriver { sb.append("\n").append(containerDiagnostics); } - // TODO HIVE-15865: Include information about pending requests, and last allocation time - // once Slider provides this information. + // TODO HIVE-15865: Include information about pending requests, and last + // allocation time once YARN Service provides this information. break; case RUNNING_ALL: sb.append("LLAP Application running with ApplicationId=") @@ -796,8 +600,8 @@ public class LlapStatusServiceDriver { SUCCESS(0), INCORRECT_USAGE(10), YARN_ERROR(20), - SLIDER_CLIENT_ERROR_CREATE_FAILED(30), - SLIDER_CLIENT_ERROR_OTHER(31), + SERVICE_CLIENT_ERROR_CREATE_FAILED(30), + SERVICE_CLIENT_ERROR_OTHER(31), LLAP_REGISTRY_ERROR(40), LLAP_JSON_GENERATION_ERROR(50), // Error in the script itself - likely caused by an incompatible change, or new functionality / states added. @@ -844,7 +648,7 @@ public class LlapStatusServiceDriver { public static void main(String[] args) { LOG.info("LLAP status invoked with arguments = {}", Arrays.toString(args)); int ret = ExitCode.SUCCESS.getInt(); - Clock clock = new SystemClock(); + Clock clock = SystemClock.getInstance(); long startTime = clock.getTime(); long lastSummaryLogTime = -1; @@ -918,7 +722,8 @@ public class LlapStatusServiceDriver { if (ret == ExitCode.SUCCESS.getInt()) { if (watchMode) { - // slider has started llap application, now if for some reason state changes to COMPLETE then fail fast + // YARN Service has started llap application, now if for some reason + // state changes to COMPLETE then fail fast if (launchingState == null && LAUNCHING_STATES.contains(currentState)) { launchingState = currentState; } @@ -984,12 +789,12 @@ public class LlapStatusServiceDriver { LOG.warn("Watch mode enabled and got YARN error. Retrying.."); numAttempts--; continue; - } else if (ret == ExitCode.SLIDER_CLIENT_ERROR_CREATE_FAILED.getInt() && watchMode) { - LOG.warn("Watch mode enabled and slider client creation failed. Retrying.."); + } else if (ret == ExitCode.SERVICE_CLIENT_ERROR_CREATE_FAILED.getInt() && watchMode) { + LOG.warn("Watch mode enabled and YARN Service client creation failed. Retrying.."); numAttempts--; continue; - } else if (ret == ExitCode.SLIDER_CLIENT_ERROR_OTHER.getInt() && watchMode) { - LOG.warn("Watch mode enabled and got slider client error. Retrying.."); + } else if (ret == ExitCode.SERVICE_CLIENT_ERROR_OTHER.getInt() && watchMode) { + LOG.warn("Watch mode enabled and got YARN Service client error. Retrying.."); numAttempts--; continue; } else if (ret == ExitCode.LLAP_REGISTRY_ERROR.getInt() && watchMode) { @@ -1066,8 +871,8 @@ public class LlapStatusServiceDriver { } private void close() { - if (sliderClient != null) { - sliderClient.stop(); + if (serviceClient != null) { + serviceClient.stop(); } if (llapRegistry != null) { llapRegistry.stop(); http://git-wip-us.apache.org/repos/asf/hive/blob/11ddb8be/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryInfo.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryInfo.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryInfo.java index d2e9396..00fed15 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryInfo.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryInfo.java @@ -184,7 +184,7 @@ public class QueryInfo { private static String createAppSpecificLocalDir(String baseDir, String applicationIdString, String user, int dagIdentifier) { // TODO This is broken for secure clusters. The app will not have permission to create these directories. - // May work via Slider - since the directory would already exist. Otherwise may need a custom shuffle handler. + // May work via YARN Service - since the directory would already exist. Otherwise may need a custom shuffle handler. // TODO This should be the process user - and not the user on behalf of whom the query is being submitted. return baseDir + File.separator + "usercache" + File.separator + user + File.separator + "appcache" + File.separator + applicationIdString + File.separator + dagIdentifier; http://git-wip-us.apache.org/repos/asf/hive/blob/11ddb8be/llap-server/src/main/resources/llap.py ---------------------------------------------------------------------- diff --git a/llap-server/src/main/resources/llap.py b/llap-server/src/main/resources/llap.py deleted file mode 100644 index 26756ce..0000000 --- a/llap-server/src/main/resources/llap.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import sys -import os -import subprocess -from resource_management import * -from os.path import dirname -from os.path import join as join_path - -class Llap(Script): - def install(self, env): - self.install_packages(env) - pass - - def configure(self, env): - import params - env.set_params(params) - - def start(self, env): - import params - env.set_params(params) - os.environ['JAVA_HOME'] = format('{java64_home}') - # this is the same as TEZ_PREFIX - os.environ['LLAP_DAEMON_HOME'] = format('{app_root}') - os.environ['LLAP_DAEMON_TMP_DIR'] = format('{app_tmp_dir}') - # this is the location where we have the llap server components (shell scripts) - os.environ['LLAP_DAEMON_BIN_HOME'] = format('{app_root}/bin') - # location containing llap-daemon-site.xml, tez and yarn configuration xmls as well. - os.environ['LLAP_DAEMON_CONF_DIR'] = format("{app_root}/conf/") - os.environ['LLAP_DAEMON_LOG_DIR'] = format("{app_log_dir}/") - os.environ['LLAP_DAEMON_LOGGER'] = format("{app_logger}") - os.environ['LLAP_DAEMON_LOG_LEVEL'] = format("{app_log_level}") - os.environ['LLAP_DAEMON_HEAPSIZE'] = format("{memory_val}") - os.environ['LLAP_DAEMON_PID_DIR'] = dirname(format("{pid_file}")) - os.environ['LLAP_DAEMON_LD_PATH'] = format('{library_path}') - os.environ['LLAP_DAEMON_OPTS'] = format('{daemon_args}') - print "Debug from LLAP python script" - print os.environ['LLAP_DAEMON_CONF_DIR'] - self.configure(env) - location = "bash -x {app_root}/bin/llapDaemon.sh start &> {app_log_dir}/shell.out" - process_cmd = format(location) - - subprocess.call(process_cmd, shell=True - ) - - def stop(self, env): - import params - env.set_params(params) - - def status(self, env): - import params - env.set_params(params) - check_process_status(params.pid_file) - -if __name__ == "__main__": - Llap().execute() - -# vim: tabstop=24 expandtab shiftwidth=4 softtabstop=4 http://git-wip-us.apache.org/repos/asf/hive/blob/11ddb8be/llap-server/src/main/resources/package.py ---------------------------------------------------------------------- diff --git a/llap-server/src/main/resources/package.py b/llap-server/src/main/resources/package.py index 21c34e9..9eb3fd7 100644 --- a/llap-server/src/main/resources/package.py +++ b/llap-server/src/main/resources/package.py @@ -10,7 +10,7 @@ import shutil import tarfile import zipfile -from templates import metainfo, appConfig, resources, runner +from templates import yarnfile, runner class LlapResource(object): def __init__(self, config): @@ -40,8 +40,7 @@ class LlapResource(object): if (not config.get("hive.llap.daemon.queue.name","")): self.queueString = "" else: - self.queueString = "--queue " - self.queueString += config["hive.llap.daemon.queue.name"] + self.queueString = config["hive.llap.daemon.queue.name"] if (not config.get("private.hive.llap.servicedriver.cluster.name")): self.clusterName="llap0" @@ -58,13 +57,13 @@ def zipdir(path, zip, prefix="."): dst = src.replace(path, prefix) zip.write(src, dst) -def slider_appconfig_global_property(arg): +def service_appconfig_global_property(arg): kv = arg.split("=") if len(kv) != 2: raise argparse.ArgumentTypeError("Value must be split into two parts separated by =") return tuple(kv) -def construct_slider_site_global_string(kvs): +def construct_service_site_global_string(kvs): if not kvs: return "" kvs = map(lambda a : a[0], kvs) @@ -76,7 +75,7 @@ def main(args): if not version: version = strftime("%d%b%Y", gmtime()) home = os.getenv("HIVE_HOME") - output = "llap-slider-%(version)s" % ({"version": version}) + output = "llap-yarn-%(version)s" % ({"version": version}) parser = argparse.ArgumentParser() parser.add_argument("--instances", type=int, default=1) parser.add_argument("--output", default=output) @@ -85,18 +84,17 @@ def main(args): parser.add_argument("--name", default="llap0") parser.add_argument("--loglevel", default="INFO") parser.add_argument("--logger", default="query-routing") - parser.add_argument("--chaosmonkey", type=int, default=0) - parser.add_argument("--slider-am-container-mb", type=int, default=1024) - parser.add_argument("--slider-appconfig-global", nargs='*', type=slider_appconfig_global_property, action='append') - parser.add_argument("--slider-keytab-dir", default="") - parser.add_argument("--slider-keytab", default="") - parser.add_argument("--slider-principal", default="") - parser.add_argument("--slider-default-keytab", dest='slider_default_keytab', action='store_true') - parser.add_argument("--slider-placement", type=int, default=4) + parser.add_argument("--service-am-container-mb", type=int, default=1024) + parser.add_argument("--service-appconfig-global", nargs='*', type=service_appconfig_global_property, action='append') + parser.add_argument("--service-keytab-dir", default="") + parser.add_argument("--service-keytab", default="") + parser.add_argument("--service-principal", default="") + parser.add_argument("--service-default-keytab", dest='service_default_keytab', action='store_true') + parser.add_argument("--service-placement", type=int, default=4) parser.add_argument("--health-percent", type=int, default=80) parser.add_argument("--health-time-window-secs", type=int, default=300) parser.add_argument("--health-init-delay-secs", type=int, default=400) - parser.set_defaults(slider_default_keytab=False) + parser.set_defaults(service_default_keytab=False) parser.add_argument("--startImmediately", dest='start_immediately', action='store_true') parser.add_argument("--javaChild", dest='java_child', action='store_true') parser.set_defaults(start_immediately=False) @@ -114,18 +112,27 @@ def main(args): input = args.input output = args.output - slider_am_jvm_heapsize = max(args.slider_am_container_mb * 0.8, args.slider_am_container_mb - 1024) - slider_keytab_dir = args.slider_keytab_dir - slider_keytab = args.slider_keytab - slider_principal = args.slider_principal + service_am_jvm_heapsize = max(args.service_am_container_mb * 0.8, args.service_am_container_mb - 1024) + service_keytab_dir = args.service_keytab_dir + service_keytab = args.service_keytab + service_principal = args.service_principal # set the defaults only if the defaults are enabled - if args.slider_default_keytab: - if not slider_keytab_dir: - slider_keytab_dir = ".slider/keytabs/llap" - if not slider_keytab: - slider_keytab = "llap.keytab" - if not slider_principal: - slider_principal = "l...@example.com" + if args.service_default_keytab: + if not service_keytab_dir: + service_keytab_dir = ".yarn/keytabs/llap" + if not service_keytab: + service_keytab = "llap.keytab" + if not service_principal: + service_principal = "l...@example.com" + service_keytab_path = service_keytab_dir + if service_keytab_path: + if service_keytab: + service_keytab_path += "/" + service_keytab + else: + service_keytab_path = service_keytab + if service_keytab_path: + service_keytab_path = "hdfs:///user/hive/" + service_keytab_path + if not input: print "Cannot find input files" sys.exit(1) @@ -140,8 +147,6 @@ def main(args): if long(max_direct_memory) > 0: daemon_args = " -XX:MaxDirectMemorySize=%s %s" % (max_direct_memory, daemon_args) daemon_args = " -Dhttp.maxConnections=%s %s" % ((max(args.instances, resource.executors) + 1), daemon_args) - # 5% container failure every monkey_interval seconds - monkey_percentage = 5 # 5% vars = { "home" : home, "version" : version, @@ -156,19 +161,15 @@ def main(args): "daemon_loglevel" : args.loglevel, "daemon_logger" : args.logger, "queue.string" : resource.queueString, - "monkey_interval" : args.chaosmonkey, - "monkey_percentage" : monkey_percentage, - "monkey_enabled" : args.chaosmonkey > 0, - "slider.am.container.mb" : args.slider_am_container_mb, - "slider_appconfig_global_append": construct_slider_site_global_string(args.slider_appconfig_global), - "slider_am_jvm_heapsize" : slider_am_jvm_heapsize, - "slider_keytab_dir" : slider_keytab_dir, - "slider_keytab" : slider_keytab, - "slider_principal" : slider_principal, - "placement" : args.slider_placement, - "health_percent": args.health_percent, - "health_time_window": args.health_time_window_secs, - "health_init_delay": args.health_init_delay_secs + "service.am.container.mb" : args.service_am_container_mb, + "service_appconfig_global_append": construct_service_site_global_string(args.service_appconfig_global), + "service_am_jvm_heapsize" : service_am_jvm_heapsize, + "service_keytab_path" : service_keytab_path, + "service_principal" : service_principal, + "placement" : args.service_placement, + "health_percent": args.health_percent, + "health_time_window": args.health_time_window_secs, + "health_init_delay": args.health_init_delay_secs } if not exists(output): @@ -180,47 +181,25 @@ def main(args): shutil.rmtree(dst) shutil.copytree(src, dst) - # Make the zip package - tmp = join(output, "tmp") - pkg = join(tmp, "package") - - src = join(home, "scripts", "llap", "slider") - dst = join(pkg, "scripts") - if exists(dst): - shutil.rmtree(dst) - shutil.copytree(src, dst) - - with open(join(tmp, "metainfo.xml"),"w") as f: - f.write(metainfo % vars) - - os.mkdir(join(pkg, "files")) + # Make the llap tarball print "%s Prepared the files" % (strftime("%H:%M:%S", gmtime())) - tarball = tarfile.open(join(pkg, "files", "llap-%s.tar.gz" % version), "w:gz") + tarball = tarfile.open(join(output, "llap-%s.tar.gz" % version), "w:gz") # recursive add + -C chdir inside tarball.add(input, "") tarball.close() - zipped = zipfile.ZipFile(join(output, "llap-%s.zip" % version), "w") - zipdir(tmp, zipped) - zipped.close() print "%s Packaged the files" % (strftime("%H:%M:%S", gmtime())) - # cleanup after making zip pkg - shutil.rmtree(tmp) - - with open(join(output, "appConfig.json"), "w") as f: - f.write(appConfig % vars) - - with open(join(output, "resources.json"), "w") as f: - f.write(resources % vars) + with open(join(output, "Yarnfile"), "w") as f: + f.write(yarnfile % vars) with open(join(output, "run.sh"), "w") as f: f.write(runner % vars) os.chmod(join(output, "run.sh"), 0700) if not args.java_child: - print "%s Prepared %s/run.sh for running LLAP on Slider" % (strftime("%H:%M:%S", gmtime()), output) + print "%s Prepared %s/run.sh for running LLAP on YARN" % (strftime("%H:%M:%S", gmtime()), output) if __name__ == "__main__": main(sys.argv[1:]) http://git-wip-us.apache.org/repos/asf/hive/blob/11ddb8be/llap-server/src/main/resources/params.py ---------------------------------------------------------------------- diff --git a/llap-server/src/main/resources/params.py b/llap-server/src/main/resources/params.py deleted file mode 100644 index 8972ba1..0000000 --- a/llap-server/src/main/resources/params.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management import * - -# server configurations -config = Script.get_config() - -app_root = config['configurations']['global']['app_root'] -java64_home = config['hostLevelParams']['java_home'] -pid_file = config['configurations']['global']['pid_file'] - -additional_cp = config['configurations']['global']['additional_cp'] -app_log_dir = config['configurations']['global']['app_log_dir'] -app_tmp_dir = config['configurations']['global']['app_tmp_dir'] -app_logger = config['configurations']['global']['app_logger'] -app_log_level = config['configurations']['global']['app_log_level'] -daemon_args = config['configurations']['global']['daemon_args'] - -port = config['configurations']['global']['listen_port'] -memory_val = config['configurations']['global']['memory_val'] -library_path = config['configurations']['global']['library_path'] - http://git-wip-us.apache.org/repos/asf/hive/blob/11ddb8be/llap-server/src/main/resources/templates.py ---------------------------------------------------------------------- diff --git a/llap-server/src/main/resources/templates.py b/llap-server/src/main/resources/templates.py index 3d747a2..00a84cf 100644 --- a/llap-server/src/main/resources/templates.py +++ b/llap-server/src/main/resources/templates.py @@ -1,138 +1,69 @@ -metainfo = """<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<metainfo> - <schemaVersion>2.0</schemaVersion> - <application> - <name>LLAP</name> - <comment>LLAP is a daemon service that works with a cache and works on SQL constructs.</comment> - <version>%(version)s</version> - <exportedConfigs>None</exportedConfigs> - <exportGroups> - <exportGroup> - <name>Servers</name> - <exports> - <export> - <name>instances</name> - <value>${LLAP_HOST}:${site.global.listen_port}</value> - </export> - </exports> - </exportGroup> - </exportGroups> - - <components> - <component> - <name>LLAP</name> - <category>MASTER</category> - <compExports>Servers-instances</compExports> - <commandScript> - <script>scripts/llap.py</script> - <scriptType>PYTHON</scriptType> - </commandScript> - </component> - </components> - - <osSpecifics> - <osSpecific> - <osType>any</osType> - <packages> - <package> - <type>tarball</type> - <name>files/llap-%(version)s.tar.gz</name> - </package> - </packages> - </osSpecific> - </osSpecifics> - - </application> -</metainfo> -""" - -appConfig = """ +yarnfile = """ { - "schema": "http://example.org/specification/v2.0.0", - "metadata": { - }, - "global": { - "application.def": ".slider/package/LLAP/llap-%(version)s.zip", - "java_home": "%(java_home)s", - "site.global.app_user": "yarn", - "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/", - "site.global.app_tmp_dir": "${AGENT_WORK_ROOT}/tmp/", - "site.global.app_logger": "%(daemon_logger)s", - "site.global.app_log_level": "%(daemon_loglevel)s", - "site.global.additional_cp": "%(hadoop_home)s", - "site.global.daemon_args": "%(daemon_args)s", - "site.global.library_path": "%(hadoop_home)s/lib/native", - "site.global.memory_val": "%(heap)d", - "site.global.pid_file": "${AGENT_WORK_ROOT}/app/run/llap-daemon.pid", - "internal.chaos.monkey.probability.amlaunchfailure": "0", - "internal.chaos.monkey.probability.containerfailure": "%(monkey_percentage)d", - "internal.chaos.monkey.interval.seconds": "%(monkey_interval)d", - "internal.chaos.monkey.enabled": "%(monkey_enabled)s"%(slider_appconfig_global_append)s - - }, - "components": { - "slider-appmaster": { - "jvm.heapsize": "%(slider_am_jvm_heapsize)dM", - "slider.hdfs.keytab.dir": "%(slider_keytab_dir)s", - "slider.am.login.keytab.name": "%(slider_keytab)s", - "slider.keytab.principal.name": "%(slider_principal)s" - } - } -} -""" - -resources = """ -{ - "schema" : "http://example.org/specification/v2.0.0", - "metadata" : { - }, - "global" : { - "yarn.log.include.patterns": ".*\\\\.done" - }, - "components": { - "slider-appmaster": { - "yarn.memory": "%(slider.am.container.mb)d", - "yarn.component.instances": "1" - }, - "LLAP": { - "yarn.role.priority": "1", - "yarn.component.instances": "%(instances)d", - "yarn.resource.normalization.enabled": "false", - "yarn.memory": "%(container.mb)d", + "name": "%(name)s", + "version": "1.0.0", + "queue": "%(queue.string)s", + "configuration": { + "properties": { + "yarn.service.rolling-log.include-pattern": ".*\\\\.done", "yarn.component.placement.policy" : "%(placement)d", "yarn.container.health.threshold.percent": "%(health_percent)d", "yarn.container.health.threshold.window.secs": "%(health_time_window)d", - "yarn.container.health.threshold.init.delay.secs": "%(health_init_delay)d" + "yarn.container.health.threshold.init.delay.secs": "%(health_init_delay)d"%(service_appconfig_global_append)s } + }, + "components": [ + { + "name": "llap", + "number_of_containers": %(instances)d, + "launch_command": "$LLAP_DAEMON_BIN_HOME/llapDaemon.sh start &> $LLAP_DAEMON_TMP_DIR/shell.out", + "artifact": { + "id": ".yarn/package/LLAP/llap-%(version)s.tar.gz", + "type": "TARBALL" + }, + "resource": { + "cpus": 1, + "memory": "%(container.mb)d" + }, + "configuration": { + "env": { + "JAVA_HOME": "%(java_home)s", + "LLAP_DAEMON_HOME": "$PWD/lib/", + "LLAP_DAEMON_TMP_DIR": "$PWD/tmp/", + "LLAP_DAEMON_BIN_HOME": "$PWD/lib/bin/", + "LLAP_DAEMON_CONF_DIR": "$PWD/lib/conf/", + "LLAP_DAEMON_LOG_DIR": "<LOG_DIR>", + "LLAP_DAEMON_LOGGER": "%(daemon_logger)s", + "LLAP_DAEMON_LOG_LEVEL": "%(daemon_loglevel)s", + "LLAP_DAEMON_HEAPSIZE": "%(heap)d", + "LLAP_DAEMON_PID_DIR": "$PWD/lib/app/run/", + "LLAP_DAEMON_LD_PATH": "%(hadoop_home)s/lib/native", + "LLAP_DAEMON_OPTS": "%(daemon_args)s", + + "APP_ROOT": "<WORK_DIR>/app/install/", + "APP_TMP_DIR": "<WORK_DIR>/tmp/" + } + } + } + ], + "kerberos_principal" : { + "principal_name" : "%(service_principal)s", + "keytab" : "%(service_keytab_path)s" + }, + "quicklinks": { + "LLAP Daemon JMX Endpoint": "http://llap-0.${SERVICE_NAME}.${USER}.${DOMAIN}:15002/jmx" } } """ -# placement policy "4" is a bit-mask -# only bit set is Slider PlacementPolicy.ANTI_AFFINITY_REQUIRED(4) +# Placement policy feature like ANTI AFFINITY is not yet merged to trunk in YARN runner = """ #!/bin/bash -e BASEDIR=$(dirname $0) -slider stop %(name)s --wait 10 || slider stop %(name)s --force --wait 30 -slider destroy %(name)s --force || slider destroy %(name)s -slider package --install --name LLAP --package $BASEDIR/llap-%(version)s.zip --replacepkg -slider create %(name)s --resources $BASEDIR/resources.json --template $BASEDIR/appConfig.json %(queue.string)s +yarn app -stop %(name)s +yarn app -destroy %(name)s +hdfs dfs -mkdir -p .yarn/package/LLAP +hdfs dfs -copyFromLocal -f $BASEDIR/llap-%(version)s.tar.gz .yarn/package/LLAP +yarn app -launch %(name)s $BASEDIR/Yarnfile """ http://git-wip-us.apache.org/repos/asf/hive/blob/11ddb8be/packaging/src/main/assembly/bin.xml ---------------------------------------------------------------------- diff --git a/packaging/src/main/assembly/bin.xml b/packaging/src/main/assembly/bin.xml index b7f6809..a9557cf 100644 --- a/packaging/src/main/assembly/bin.xml +++ b/packaging/src/main/assembly/bin.xml @@ -206,7 +206,7 @@ <include>**/*.py</include> <include>**/*.xml</include> </includes> - <outputDirectory>scripts/llap/slider</outputDirectory> + <outputDirectory>scripts/llap/yarn</outputDirectory> </fileSet> <fileSet> http://git-wip-us.apache.org/repos/asf/hive/blob/11ddb8be/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index fa80637..19a9b71 100644 --- a/pom.xml +++ b/pom.xml @@ -198,7 +198,6 @@ <ST4.version>4.0.4</ST4.version> <storage-api.version>2.6.1-SNAPSHOT</storage-api.version> <tez.version>0.9.1</tez.version> - <slider.version>0.92.0-incubating</slider.version> <super-csv.version>2.2.0</super-csv.version> <spark.version>2.3.0</spark.version> <scala.binary.version>2.11</scala.binary.version>