YARN-7190. Ensure only NM classpath in 2.x gets TSv2 related hbase jars, not the user classpath. Contributed by Varun Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09d996fd Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09d996fd Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09d996fd Branch: refs/heads/HDFS-9806 Commit: 09d996fdd429a85822a06ab87f0e2322d0d7ca68 Parents: 890d3d0 Author: Rohith Sharma K S <[email protected]> Authored: Fri Dec 15 21:50:28 2017 +0530 Committer: Rohith Sharma K S <[email protected]> Committed: Fri Dec 15 21:50:28 2017 +0530 ---------------------------------------------------------------------- .../resources/assemblies/hadoop-yarn-dist.xml | 27 ++++++++++++++ hadoop-yarn-project/hadoop-yarn/bin/yarn | 6 ++++ hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 6 ++++ .../hadoop-yarn-server-resourcemanager/pom.xml | 1 + .../pom.xml | 23 ++++++++++++ .../hadoop-yarn-server-timelineservice/pom.xml | 1 + hadoop-yarn-project/pom.xml | 38 +++++++++++++++++--- 7 files changed, 97 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/09d996fd/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml ---------------------------------------------------------------------- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index 83633ac..a77dd20 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -211,10 +211,37 @@ <include>**/*</include> </includes> </fileSet> + <!-- Copy dependecies from hadoop-yarn-server-timelineservice as well --> + <fileSet> + <directory>hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/target/lib</directory> + <outputDirectory>share/hadoop/${hadoop.component}/timelineservice/lib</outputDirectory> + </fileSet> </fileSets> <moduleSets> <moduleSet> + <includes> + <include>org.apache.hadoop:hadoop-yarn-server-timelineservice</include> + <include>org.apache.hadoop:hadoop-yarn-server-timelineservice-hbase</include> + </includes> + <binaries> + <outputDirectory>share/hadoop/${hadoop.component}/timelineservice</outputDirectory> + <includeDependencies>false</includeDependencies> + <unpack>false</unpack> + </binaries> + </moduleSet> + <moduleSet> + <includes> + <include>org.apache.hadoop:hadoop-yarn-server-timelineservice-hbase-tests</include> + </includes> + <binaries> + <outputDirectory>share/hadoop/${hadoop.component}/timelineservice/test</outputDirectory> + <includeDependencies>false</includeDependencies> + <unpack>false</unpack> + </binaries> + </moduleSet> + <moduleSet> <excludes> + <exclude>org.apache.hadoop:hadoop-yarn-server-timelineservice*</exclude> <exclude>org.apache.hadoop:hadoop-yarn-ui</exclude> </excludes> <binaries> http://git-wip-us.apache.org/repos/asf/hadoop/blob/09d996fd/hadoop-yarn-project/hadoop-yarn/bin/yarn ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 9a1dc19..69afe6f 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -121,6 +121,8 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" ;; nodemanager) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/*" + hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.nodemanager.NodeManager' # Backwards compatibility if [[ -n "${YARN_NODEMANAGER_HEAPSIZE}" ]]; then @@ -145,6 +147,8 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" ;; resourcemanager) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/*" + hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager' # Backwards compatibility if [[ -n "${YARN_RESOURCEMANAGER_HEAPSIZE}" ]]; then @@ -178,6 +182,8 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" ;; timelinereader) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/*" + hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer' ;; timelineserver) http://git-wip-us.apache.org/repos/asf/hadoop/blob/09d996fd/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd index fed3d90..e1ac112 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd @@ -219,6 +219,8 @@ goto :eof :resourcemanager set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\rm-config\log4j.properties + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\* + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\lib\* set CLASS=org.apache.hadoop.yarn.server.resourcemanager.ResourceManager set YARN_OPTS=%YARN_OPTS% %YARN_RESOURCEMANAGER_OPTS% if defined YARN_RESOURCEMANAGER_HEAPSIZE ( @@ -248,6 +250,8 @@ goto :eof :timelinereader set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\timelineserver-config\log4j.properties + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\* + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\lib\* set CLASS=org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer set YARN_OPTS=%YARN_OPTS% %YARN_TIMELINEREADER_OPTS% goto :eof @@ -260,6 +264,8 @@ goto :eof :nodemanager set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\nm-config\log4j.properties + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\* + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\lib\* set CLASS=org.apache.hadoop.yarn.server.nodemanager.NodeManager set YARN_OPTS=%YARN_OPTS% -server %HADOOP_NODEMANAGER_OPTS% if defined YARN_NODEMANAGER_HEAPSIZE ( http://git-wip-us.apache.org/repos/asf/hadoop/blob/09d996fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml index 25a201c..a4d7afc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml @@ -191,6 +191,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-server-timelineservice</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> http://git-wip-us.apache.org/repos/asf/hadoop/blob/09d996fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml index f200e9a..3739301 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml @@ -57,11 +57,13 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-annotations</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> + <scope>provided</scope> </dependency> <!-- 'mvn dependency:analyze' fails to detect use of this dependency --> @@ -75,16 +77,19 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-api</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-common</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId> + <scope>provided</scope> </dependency> <dependency> @@ -121,6 +126,7 @@ <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-server</artifactId> + <scope>provided</scope> <exclusions> <exclusion> <groupId>org.apache.hadoop</groupId> @@ -186,6 +192,23 @@ </additionnalDependencies> </configuration> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <includeScope>runtime</includeScope> + <excludeGroupIds>org.slf4j,org.apache.hadoop,com.github.stephenc.findbugs</excludeGroupIds> + <outputDirectory>${project.build.directory}/lib</outputDirectory> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> </project> http://git-wip-us.apache.org/repos/asf/hadoop/blob/09d996fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml index 5cbfbf5..4236c51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml @@ -38,6 +38,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> + <scope>provided</scope> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/hadoop/blob/09d996fd/hadoop-yarn-project/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml index 1b3c5f0..3ef9c45 100644 --- a/hadoop-yarn-project/pom.xml +++ b/hadoop-yarn-project/pom.xml @@ -76,10 +76,6 @@ </dependency> <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-server-timelineservice-hbase</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-server-router</artifactId> </dependency> <dependency> @@ -167,12 +163,44 @@ </execution> </executions> </plugin> + <!-- Remove duplicate dependencies which exist in both yarn/lib folder and yarn/timelineservice/lib folder --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <id>dist</id> + <phase>prepare-package</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <target> + <echo file="${project.build.directory}/remove-duplicate-timelineservice-jars.sh"> + yarnjardir="${project.build.directory}/${project.artifactId}-${project.version}/share/hadoop/yarn" + yarnlibdir="${yarnjardir}/lib/" + cd "${yarnjardir}/timelineservice/lib" + for jar in `ls` + do + if [ -f "${yarnlibdir}""${jar}" ];then + rm -rf ${jar} + echo "Removing duplicate jar $jar from share/hadoop/yarn/timelineservice/lib folder" + fi + done + </echo> + <exec executable="${shell-executable}" dir="${project.build.directory}" failonerror="true"> + <arg line="./remove-duplicate-timelineservice-jars.sh"/> + </exec> + </target> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> </profile> </profiles> - <reporting> <plugins> <plugin> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
