YARN-5690. Integrate native services modules into maven build. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/00c38659 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/00c38659 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/00c38659 Branch: refs/heads/yarn-native-services Commit: 00c38659e43d6b99de43212470ed7a4c8d45d48a Parents: 432974a Author: Gour Saha <gourks...@apache.org> Authored: Thu Oct 27 08:50:36 2016 -0700 Committer: Jian He <jia...@apache.org> Committed: Wed Dec 7 13:00:06 2016 -0800 ---------------------------------------------------------------------- .../resources/assemblies/hadoop-yarn-dist.xml | 26 ++++++++ .../assemblies/hadoop-yarn-services-api.xml | 36 +++++++++++ .../assemblies/hadoop-yarn-slider-dist.xml | 30 +++++++++ hadoop-project/pom.xml | 6 ++ hadoop-yarn-project/hadoop-yarn/bin/yarn | 30 +++++++++ .../hadoop-yarn-services-api/pom.xml | 44 +++++++++++-- .../conf/slideram-log4j.properties | 68 ++++++++++++++++++++ .../hadoop-yarn-slider-core/pom.xml | 38 +++++++++++ .../org/apache/slider/client/SliderClient.java | 29 +++++++-- .../org/apache/slider/common/SliderKeys.java | 2 +- .../apache/slider/common/tools/SliderUtils.java | 48 ++++++++++---- .../providers/agent/AgentClientProvider.java | 3 +- .../slideram/SliderAMClientProvider.java | 15 +++-- .../TestPublishedConfigurationOutputter.java | 10 ++- 14 files changed, 343 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/00c38659/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml ---------------------------------------------------------------------- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index c3f459c..552087c 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -86,6 +86,32 @@ </includes> </fileSet> <fileSet> + <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/target</directory> + <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory> + <includes> + <include>*-sources.jar</include> + </includes> + </fileSet> + <fileSet> + <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf</directory> + <outputDirectory>etc/hadoop</outputDirectory> + </fileSet> + <fileSet> + <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/target/hadoop-yarn-slider-core-${project.version}</directory> + <outputDirectory>/share/hadoop/${hadoop.component}/lib/slider</outputDirectory> + </fileSet> + <fileSet> + <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target</directory> + <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory> + <includes> + <include>*-sources.jar</include> + </includes> + </fileSet> + <fileSet> + <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target/hadoop-yarn-services-api-${project.version}</directory> + <outputDirectory>/share/hadoop/${hadoop.component}/lib/services-api</outputDirectory> + </fileSet> + <fileSet> <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/target</directory> <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory> <includes> http://git-wip-us.apache.org/repos/asf/hadoop/blob/00c38659/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml ---------------------------------------------------------------------- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml new file mode 100644 index 0000000..589f724 --- /dev/null +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml @@ -0,0 +1,36 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.01 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd";> + <id>hadoop-yarn-services-api-dist</id> + <formats> + <format>dir</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + <dependencySets> + <dependencySet> + <useProjectArtifact>false</useProjectArtifact> + <includes> + <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base</include> + <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider</include> + <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations</include> + <include>io.swagger:swagger-annotations</include> + </includes> + </dependencySet> + </dependencySets> +</assembly> http://git-wip-us.apache.org/repos/asf/hadoop/blob/00c38659/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml ---------------------------------------------------------------------- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml new file mode 100644 index 0000000..5de45a9 --- /dev/null +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml @@ -0,0 +1,30 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.01 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd";> + <id>hadoop-yarn-slider-dist</id> + <formats> + <format>dir</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + <dependencySets> + <dependencySet> + <useProjectArtifact>false</useProjectArtifact> + </dependencySet> + </dependencySets> +</assembly> http://git-wip-us.apache.org/repos/asf/hadoop/blob/00c38659/hadoop-project/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index a2b6e25..96b3d86 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -343,6 +343,12 @@ <dependency> <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-slider-core</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-jobclient</artifactId> <version>${project.version}</version> <type>test-jar</type> http://git-wip-us.apache.org/repos/asf/hadoop/blob/00c38659/hadoop-yarn-project/hadoop-yarn/bin/yarn ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 804fd1a..26d54b8 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -47,7 +47,9 @@ function hadoop_usage hadoop_add_subcommand "resourcemanager" "run the ResourceManager" hadoop_add_subcommand "rmadmin" "admin tools" hadoop_add_subcommand "scmadmin" "SharedCacheManager admin tools" + hadoop_add_subcommand "services-api" "run slider services api" hadoop_add_subcommand "sharedcachemanager" "run the SharedCacheManager daemon" + hadoop_add_subcommand "slider" "run a slider app" hadoop_add_subcommand "timelinereader" "run the timeline reader server" hadoop_add_subcommand "timelineserver" "run the timeline server" hadoop_add_subcommand "top" "view cluster information" @@ -142,10 +144,38 @@ function yarncmd_case scmadmin) HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin' ;; + services-api) + HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*' + hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services-api"'/*' + HADOOP_CLASSNAME='org.apache.hadoop.yarn.services.webapp.ApplicationApiWebApp' + hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS" + HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS} \ +-Dslider.libdir=${HADOOP_YARN_HOME}/${YARN_DIR},\ +${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\ +${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\ +${HADOOP_HDFS_HOME}/${HDFS_DIR},\ +${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\ +${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\ +${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" + ;; sharedcachemanager) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager' ;; + slider) + hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*' + HADOOP_CLASSNAME='org.apache.slider.Slider' + hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS" + HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS} \ +-Dslider.libdir=${HADOOP_YARN_HOME}/${YARN_DIR},\ +${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\ +${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\ +${HADOOP_HDFS_HOME}/${HDFS_DIR},\ +${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\ +${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\ +${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" + ;; timelinereader) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer' http://git-wip-us.apache.org/repos/asf/hadoop/blob/00c38659/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml index 78b7855..c198f28 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml @@ -159,12 +159,6 @@ <dependency> <groupId>com.fasterxml.jackson.jaxrs</groupId> <artifactId>jackson-jaxrs-json-provider</artifactId> - <exclusions> - <exclusion> - <groupId>com.fasterxml.jackson.jaxrs</groupId> - <artifactId>jackson-jaxrs-base</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> <groupId>org.mockito</groupId> @@ -192,6 +186,44 @@ </dependencies> <profiles> + <profile> + <id>dist</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-assemblies</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + <executions> + <execution> + <id>dist</id> + <phase>prepare-package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <appendAssemblyId>false</appendAssemblyId> + <attach>false</attach> + <finalName>${project.artifactId}-${project.version}</finalName> + <descriptorRefs> + <descriptorRef>hadoop-yarn-services-api</descriptorRef> + </descriptorRefs> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> <profile> <id>rat</id> http://git-wip-us.apache.org/repos/asf/hadoop/blob/00c38659/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties new file mode 100644 index 0000000..333859e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is the log4j configuration for Slider Application Master + +# Log rotation based on size (100KB) with a max of 10 backup files +log4j.rootLogger=INFO, amlog +log4j.threshhold=ALL +log4j.appender.amlog=org.apache.log4j.RollingFileAppender +log4j.appender.amlog.layout=org.apache.log4j.PatternLayout +log4j.appender.amlog.File=${LOG_DIR}/slider.log +log4j.appender.amlog.MaxFileSize=1MB +log4j.appender.amlog.MaxBackupIndex=10 + +# log layout skips stack-trace creation operations by avoiding line numbers and method +log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n + +# debug edition is much more expensive +#log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n + +# configure stderr +# set the conversion pattern of stderr +# Print the date in ISO 8601 format +log4j.appender.stderr=org.apache.log4j.ConsoleAppender +log4j.appender.stderr.Target=System.err +log4j.appender.stderr.layout=org.apache.log4j.PatternLayout +log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n + +log4j.appender.subprocess=org.apache.log4j.ConsoleAppender +log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout +log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n +#log4j.logger.org.apache.slider.yarn.appmaster.SliderAppMasterer.master=INFO,subprocess + +# for debugging Slider +#log4j.logger.org.apache.slider=DEBUG + +# uncomment to debug service lifecycle issues +#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG +#log4j.logger.org.apache.hadoop.yarn.service=DEBUG + +# uncomment for YARN operations +#log4j.logger.org.apache.hadoop.yarn.client=DEBUG + +# uncomment this to debug security problems +#log4j.logger.org.apache.hadoop.security=DEBUG + +#crank back on some noise +log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR +log4j.logger.org.apache.hadoop.hdfs=WARN +log4j.logger.org.apache.hadoop.hdfs.shortcircuit=ERROR + +log4j.logger.org.apache.zookeeper=WARN +log4j.logger.org.apache.curator.framework.state=ERROR +log4j.logger.org.apache.curator.framework.imps=WARN http://git-wip-us.apache.org/repos/asf/hadoop/blob/00c38659/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml index d778f44..a2c67c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml @@ -332,6 +332,44 @@ <profiles> <profile> + <id>dist</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-assemblies</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + <executions> + <execution> + <id>dist</id> + <phase>prepare-package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <appendAssemblyId>false</appendAssemblyId> + <attach>false</attach> + <finalName>${project.artifactId}-${project.version}</finalName> + <descriptorRefs> + <descriptorRef>hadoop-yarn-slider-dist</descriptorRef> + </descriptorRefs> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + <profile> <id>compile-protobuf</id> <build> <plugins> http://git-wip-us.apache.org/repos/asf/hadoop/blob/00c38659/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 4b546cd..d1f88c5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -2162,6 +2162,24 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe new File(confDir, SliderKeys.LOG4J_SERVER_PROP_FILENAME); hasServerLog4jProperties = log4jserver.isFile(); } + if (!hasServerLog4jProperties) { + // check for log4j properties in hadoop conf dir + String hadoopConfDir = System.getenv(ApplicationConstants.Environment + .HADOOP_CONF_DIR.name()); + if (hadoopConfDir != null) { + File localFile = new File(hadoopConfDir, SliderKeys + .LOG4J_SERVER_PROP_FILENAME); + if (localFile.exists()) { + Path localFilePath = createLocalPath(localFile); + remoteConfPath = new Path(clusterDirectory, + SliderKeys.SUBMITTED_CONF_DIR); + Path remoteFilePath = new Path(remoteConfPath, SliderKeys + .LOG4J_SERVER_PROP_FILENAME); + copy(config, localFilePath, remoteFilePath); + hasServerLog4jProperties = true; + } + } + } // the assumption here is that minimr cluster => this is a test run // and the classpath can look after itself @@ -2300,7 +2318,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // enable asserts commandLine.enableJavaAssertions(); - // if the conf dir has a log4j-server.properties, switch to that + // if the conf dir has a slideram-log4j.properties, switch to that if (hasServerLog4jProperties) { commandLine.sysprop(SYSPROP_LOG4J_CONFIGURATION, LOG4J_SERVER_PROP_FILENAME); commandLine.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR); @@ -4471,14 +4489,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return EXIT_SUCCESS; } - String libDir = System.getProperty(SliderKeys.PROPERTY_LIB_DIR); - if (isSet(libDir)) { - File srcFolder = new File(libDir); + String[] libDirs = SliderUtils.getLibDirs(); + if (libDirs.length > 0) { File tempLibTarGzipFile = File.createTempFile( SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME + "_", SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT); - // copy all jars except slider-core-<version>.jar - tarGzipFolder(srcFolder, tempLibTarGzipFile, createJarFilter()); + // copy all jars + tarGzipFolder(libDirs, tempLibTarGzipFile, createJarFilter()); log.info("Uploading dependency for AM (version {}) from {} to {}", version, tempLibTarGzipFile.toURI(), dependencyLibTarGzip.toUri()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/00c38659/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index 1484ee3..3d25d33 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -182,7 +182,7 @@ public interface SliderKeys extends SliderXmlConfKeys { /** * Slider AM log4j file name : {@value} */ - String LOG4J_SERVER_PROP_FILENAME = "log4j-server.properties"; + String LOG4J_SERVER_PROP_FILENAME = "slideram-log4j.properties"; /** * Standard log4j file name : {@value} http://git-wip-us.apache.org/repos/asf/hadoop/blob/00c38659/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index f773982..713cd02 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -25,6 +25,7 @@ import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; import org.apache.commons.io.output.ByteArrayOutputStream; +import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; @@ -1399,6 +1400,22 @@ public final class SliderUtils { } } + public static String getLibDir() { + String[] libDirs = getLibDirs(); + if (libDirs == null || libDirs.length == 0) { + return null; + } + return libDirs[0]; + } + + public static String[] getLibDirs() { + String libDirStr = System.getProperty(SliderKeys.PROPERTY_LIB_DIR); + if (isUnset(libDirStr)) { + return ArrayUtils.EMPTY_STRING_ARRAY; + } + return StringUtils.split(libDirStr, ','); + } + /** * Submit a JAR containing a specific class and map it * @param providerResources provider map to build up @@ -1962,31 +1979,34 @@ public final class SliderUtils { /** * Given a source folder create a tar.gz file * - * @param srcFolder + * @param libDirs * @param tarGzipFile * * @throws IOException */ - public static void tarGzipFolder(File srcFolder, File tarGzipFile, + public static void tarGzipFolder(String[] libDirs, File tarGzipFile, FilenameFilter filter) throws IOException { - log.info("Tar-gzipping folder {} to {}", srcFolder.getAbsolutePath(), + log.info("Tar-gzipping folders {} to {}", libDirs, tarGzipFile.getAbsolutePath()); - List<String> files = new ArrayList<>(); - generateFileList(files, srcFolder, srcFolder, true, filter); try(TarArchiveOutputStream taos = new TarArchiveOutputStream(new GZIPOutputStream( new BufferedOutputStream(new FileOutputStream(tarGzipFile))))) { - for (String file : files) { - File srcFile = new File(srcFolder, file); - TarArchiveEntry tarEntry = new TarArchiveEntry( - srcFile, file); - taos.putArchiveEntry(tarEntry); - try(FileInputStream in = new FileInputStream(srcFile)) { - org.apache.commons.io.IOUtils.copy(in, taos); + for (String libDir : libDirs) { + File srcFolder = new File(libDir); + List<String> files = new ArrayList<>(); + generateFileList(files, srcFolder, srcFolder, true, filter); + for (String file : files) { + File srcFile = new File(srcFolder, file); + TarArchiveEntry tarEntry = new TarArchiveEntry( + srcFile, file); + taos.putArchiveEntry(tarEntry); + try(FileInputStream in = new FileInputStream(srcFile)) { + org.apache.commons.io.IOUtils.copy(in, taos); + } + taos.flush(); + taos.closeArchiveEntry(); } - taos.flush(); - taos.closeArchiveEntry(); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/00c38659/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java index fdc5be1..7ca469f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java @@ -562,8 +562,7 @@ public class AgentClientProvider extends AbstractClientProvider } private void expandAgentTar(File agentPkgDir) throws IOException { - String libDirProp = - System.getProperty(PROPERTY_LIB_DIR); + String libDirProp = SliderUtils.getLibDir(); File tarFile = new File(libDirProp, AGENT_TAR); expandTar(tarFile, agentPkgDir); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/00c38659/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java index e5430f2..b58d3aa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java @@ -184,19 +184,20 @@ public class SliderAMClientProvider extends AbstractClientProvider libdir, miniClusterTestRun); - String libDirProp = - System.getProperty(SliderKeys.PROPERTY_LIB_DIR); log.info("Loading all dependencies for AM."); // If slider.tar.gz is available in hdfs use it, else upload all jars Path dependencyLibTarGzip = fileSystem.getDependencyTarGzip(); if (fileSystem.isFile(dependencyLibTarGzip)) { SliderUtils.putAmTarGzipAndUpdate(providerResources, fileSystem); } else { - ProviderUtils.addAllDependencyJars(providerResources, - fileSystem, - tempPath, - libdir, - libDirProp); + for (String libDirProp : SliderUtils.getLibDirs()) { + ProviderUtils.addAllDependencyJars(providerResources, + fileSystem, + tempPath, + libdir, + libDirProp); + + } } addKeytabResourceIfNecessary(fileSystem, instanceDescription, http://git-wip-us.apache.org/repos/asf/hadoop/blob/00c38659/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java index 3706182..56b84e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java @@ -82,16 +82,14 @@ public class TestPublishedConfigurationOutputter { String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)", ""); - assert output.contains( - "<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>"); + assert output.contains("<name>key1</name><value>val1</value>"); File file = tmpDir.newFile(); configurationOutputter.save(file); assert FileUtils.readFileToString(file, Charsets.UTF_8) .replaceAll("( |\\r|\\n)", "") - .contains( - "<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>"); + .contains("<name>key1</name><value>val1</value>"); } @Test @@ -103,14 +101,14 @@ public class TestPublishedConfigurationOutputter { String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)", ""); - assert output.contains("<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>"); + assert output.contains("<name>key1</name><value>val1</value>"); File file = tmpDir.newFile(); configurationOutputter.save(file); assert FileUtils.readFileToString(file, Charsets.UTF_8) .replaceAll("( |\\r|\\n)", "") - .contains( "<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>"); + .contains("<name>key1</name><value>val1</value>"); } @Test --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
