This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/fluo-yarn.git
The following commit(s) were added to refs/heads/master by this push: new bc223f6 Convert multi-module build to single module (#17) bc223f6 is described below commit bc223f601ab476195c0d87852ba64b270029e523 Author: Christopher Tubbs <ctubb...@apache.org> AuthorDate: Wed Feb 28 19:56:52 2018 -0500 Convert multi-module build to single module (#17) * Simplify build by consolidating the tarball assembly and the jar into a single module Maven build * Update copyright date * Update travis build configuration * Simplify assembly by removing unneeded configs * Remove unused profiles and plugin config in the POM * Add script to easily create release candidates * Add script for creating release candidates * Add missing scm and issueManagement sections to POM --- .travis.yml | 23 +- NOTICE | 2 +- contrib/create-release-candidate.sh | 287 +++++++++++++++++++++ core/pom.xml | 60 ----- distribution/pom.xml | 90 ------- pom.xml | 159 +++++------- {distribution/src => src}/main/assembly/bin.xml | 30 +-- .../main/assembly/contents}/bin/fluo-yarn | 4 +- .../main/assembly/contents}/conf/fluo-yarn-env.sh | 0 .../assembly/contents}/conf/fluo-yarn.properties | 0 .../main/assembly/contents}/conf/log4j.properties | 0 .../main/assembly/contents}/conf/logback.xml | 0 .../main/assembly/contents}/lib/fetch.sh | 0 .../java/org/apache/fluo/yarn}/FluoYarnEnv.java | 2 +- .../org/apache/fluo/yarn}/FluoYarnLauncher.java | 2 +- .../org/apache/fluo/yarn}/FluoYarnProperties.java | 2 +- 16 files changed, 385 insertions(+), 276 deletions(-) diff --git a/.travis.yml b/.travis.yml index bb72f1c..726f507 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,18 +3,29 @@ # copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance with the License. You may obtain a # copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software distributed under the License # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing permissions and limitations under # the License. - language: java -jdk: - - oraclejdk8 -script: mvn -U verify +notifications: + irc: + channels: + - "chat.freenode.net#fluo" + use_notice: true + on_success: change + on_failure: always + skip_join: true cache: directories: - $HOME/.m2 +install: echo NOOP Skipping pre-fetch of Maven dependencies +jdk: + - openjdk8 +env: + - ADDITIONAL_MAVEN_OPTS= +script: + - mvn clean verify javadoc:jar $ADDITIONAL_MAVEN_OPTS diff --git a/NOTICE b/NOTICE index 0a27673..d974d76 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ Apache Fluo YARN launcher -Copyright 2017 The Apache Software Foundation. +Copyright 2018 The Apache Software Foundation. This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/contrib/create-release-candidate.sh b/contrib/create-release-candidate.sh new file mode 100755 index 0000000..4b46543 --- /dev/null +++ b/contrib/create-release-candidate.sh @@ -0,0 +1,287 @@ +#! /usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd "$(dirname "$0")/.." || exit 1 +scriptname=$(basename "$0") + +# check for gpg2 +hash gpg2 2>/dev/null && gpgCommand=gpg2 || gpgCommand=gpg + +# check if running in a color terminal +terminalSupportsColor() { + local c; c=$(tput colors 2>/dev/null) || c=-1 + [[ -t 1 ]] && [[ $c -ge 8 ]] +} +terminalSupportsColor && doColor=1 || doColor=0 + +color() { local c; c=$1; shift; [[ $doColor -eq 1 ]] && echo -e "\\e[0;${c}m${*}\\e[0m" || echo "$@"; } +red() { color 31 "$@"; } +green() { color 32 "$@"; } +yellow() { color 33 "$@"; } + +fail() { echo -e ' ' "$@"; exit 1; } +runLog() { local o; o=$1 && shift && echo "$(green Running) $(yellow "$@" '>>' "$o")" && echo Running "$@" >> "$o" && eval "$@" >> "$o"; } +run() { echo "$(green Running) $(yellow "$@")" && eval "$@"; } +runOrFail() { run "$@" || fail "$(yellow "$@")" "$(red failed)"; } + +currentBranch() { local b; b=$(git symbolic-ref -q HEAD) && echo "${b##refs/heads/}"; } + +cacheGPG() { + # make sure gpg agent has key cached + # first clear cache, to reset timeouts (best attempt) + { hash gpg-connect-agent && gpg-connect-agent reloadagent /bye; } &>/dev/null + # TODO prompt for key instead of using default? + local TESTFILE; TESTFILE=$(mktemp --tmpdir "${USER}-gpgTestFile-XXXXXXXX.txt") + [[ -r $TESTFILE ]] && "$gpgCommand" --sign "${TESTFILE}" && rm -f "${TESTFILE}" "${TESTFILE}.gpg" +} + +prompter() { + # $1 description; $2 pattern to validate against + local x + read -r -p "Enter the $1: " x + until eval "[[ \$x =~ ^$2\$ ]]"; do + echo " $(red "$x") is not a proper $1" 1>&2 + read -r -p "Enter the $1: " x + done + echo "$x" +} + +pretty() { local f; f=$1; shift; git log "--pretty=tformat:$f" "$@"; } +gitCommits() { pretty %H "$@"; } +gitCommit() { gitCommits -n1 "$@"; } +gitSubject() { pretty %s "$@"; } + +createEmail() { + # $1 version (optional); $2 rc seqence num (optional); $3 staging repo num (optional) + local ver; [[ -n $1 ]] && ver=$1 || ver=$(prompter 'version to be released (eg. x.y.z)' '[0-9]+[.][0-9]+[.][0-9]+') + local rc; [[ -n $2 ]] && rc=$2 || rc=$(prompter 'release candidate sequence number (eg. 1, 2, etc.)' '[0-9]+') + local stagingrepo; [[ -n $3 ]] && stagingrepo=$3 || stagingrepo=$(prompter 'staging repository number from https://repository.apache.org/#stagingRepositories' '[0-9]+') + + local branch; branch=$ver-rc$rc + local commit; commit=$(gitCommit "$branch") || exit 1 + local tag; tag=rel/fluo-yarn-$ver + echo + yellow "IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!!" + echo + echo " Don't forget to push a branch named $(green "$branch") with" + echo " its head at $(green "${commit:0:7}") so others can review using:" + echo " $(green "git push origin ${commit:0:7}:refs/heads/$branch")" + echo + echo " Remember, $(red DO NOT PUSH) the $(red "$tag") tag until after the vote" + echo " passes and the tag is re-made with a gpg signature using:" + echo " $(red "git tag -f -m 'Apache Fluo Recipes $ver' -s $tag ${commit:0:7}")" + echo + yellow "IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!!" + echo + read -r -s -p 'Press Enter to generate the [VOTE] email...' + echo 1>&2 + + # compute the date with a buffer of 30 minutes + local votedate; votedate=$(date -d "+3 days 30 minutes" "+%s") + # round back to the previous half-hour + local halfhour; halfhour=$((votedate - (votedate % 1800))) + votedate=$(date -u -d"1970-01-01 $halfhour seconds UTC") + export TZ="America/New_York" + local edtvotedate; edtvotedate=$(date -d"1970-01-01 $halfhour seconds UTC") + export TZ="America/Los_Angeles" + local pdtvotedate; pdtvotedate=$(date -d"1970-01-01 $halfhour seconds UTC") + + local fingerprint; fingerprint=$("$gpgCommand" --list-secret-keys --with-colons --with-fingerprint 2>/dev/null | awk -F: '$1 == "fpr" {print $10}') + [[ -z $fingerprint ]] && fingerprint="UNSPECIFIED" + + cat <<EOF +$(yellow '============================================================') +Subject: $(green [VOTE] Apache Fluo Recipes "$branch") +$(yellow '============================================================') +Fluo Developers, + +Please consider the following candidate for Fluo Recipes $(green "$ver"). + +Git Commit: + $(green "$commit") +Branch: + $(green "$branch") + +If this vote passes, a gpg-signed tag will be created using: + $(green "git tag -f -m 'Apache Fluo Recipes $ver' -s $tag") \\ + $(green "$commit") +Staging repo: $(green "https://repository.apache.org/content/repositories/orgapachefluo-$stagingrepo") +Source (official release artifact): $(green "https://repository.apache.org/content/repositories/orgapachefluo-$stagingrepo/org/apache/fluo/fluo-yarn/$ver/fluo-yarn-${ver}-source-release.tar.gz") +Binary: $(green "https://repository.apache.org/content/repositories/orgapachefluo-$stagingrepo/org/apache/fluo/fluo-yarn/$ver/fluo-yarn-${ver}-bin.tar.gz") +(Append ".sha1", ".md5", or ".asc" to download the signature/hash for a given artifact.) + +All artifacts were built and staged with: + mvn release:prepare && mvn release:perform + +Signing keys are available at https://www.apache.org/dist/fluo/KEYS +(Expected fingerprint: $(green "$fingerprint")) + +Release notes (in progress) can be found at: $(green "https://fluo.apache.org/.../$ver") + +Please vote one of: +[ ] +1 - I have verified and accept... +[ ] +0 - I have reservations, but not strong enough to vote against... +[ ] -1 - Because..., I do not accept... +... these artifacts as the $(green "$ver") release of Apache Fluo Recipes. + +This vote will end on $(green "$votedate") +($(green "$edtvotedate") / $(green "$pdtvotedate")) + +Thanks! + +P.S. Hint: download the whole staging repo with + wget -erobots=off -r -l inf -np -nH \\ + $(green "https://repository.apache.org/content/repositories/orgapachefluo-$stagingrepo/") + # note the trailing slash is needed +$(yellow '============================================================') +EOF +} + +cleanUpAndFail() { + # $1 command; $2 log; $3 original branch; $4 next branch + echo " Failure in $(red "$1")!" + echo " Check output in $(yellow "$2")" + echo " Initiating clean up steps..." + + run git checkout "$3" + + # pre-populate branches with expected next branch; de-duplicate later + local branches; branches=("$4") + local tags; tags=() + local x; local y + for x in $(gitCommits "${cBranch}..${nBranch}"); do + for y in $(git branch --contains "$x" | cut -c3-); do + branches=("${branches[@]}" "$y") + done + for y in $(git tag --contains "$x"); do + tags=("${tags[@]}" "$y") + done + done + + # de-duplicate branches + local a + branches=($(printf "%s\n" "${branches[@]}" | sort -u)) + for x in "${branches[@]}"; do + echo "Do you wish to clean up (delete) the branch $(yellow "$x")?" + a=$(prompter "letter 'y' or 'n'" '[yn]') + [[ $a == 'y' ]] && git branch -D "$x" + done + for x in "${tags[@]}"; do + echo "Do you wish to clean up (delete) the tag $(yellow "$x")?" + a=$(prompter "letter 'y' or 'n'" '[yn]') + [[ $a == 'y' ]] && git tag -d "$x" + done + exit 1 +} + +createReleaseCandidate() { + yellow "WARNING!! WARNING!! WARNING!! WARNING!! WARNING!! WARNING!!" + echo + echo " This will modify your local git repository by creating" + echo " branches and tags. Afterwards, you may need to perform" + echo " some manual steps to complete the release or to rollback" + echo " in the case of failure." + echo + yellow "WARNING!! WARNING!! WARNING!! WARNING!! WARNING!! WARNING!!" + echo + + local extraReleaseArgs; extraReleaseArgs=("$@") + if [[ ${#extraReleaseArgs[@]} -ne 0 ]]; then + red "CAUTION!! Extra release args may create a non-standard release!!" + red "You added '${extraReleaseArgs[*]}'" + fi + [[ ${#extraReleaseArgs[@]} -eq 0 ]] && [[ $gpgCommand != 'gpg' ]] && extraReleaseArgs=("-Dgpg.executable=$gpgCommand") + extraReleaseArgs="-DextraReleaseArguments='${extraReleaseArgs[*]}'" + + local ver + ver=$(xmllint --shell pom.xml <<<'xpath /*[local-name()="project"]/*[local-name()="version"]/text()' | grep content= | cut -f2 -d=) + ver=${ver%%-SNAPSHOT} + echo "Building release candidate for version: $(green "$ver")" + local tag; tag=rel/fluo-yarn-$ver + + local cBranch; cBranch=$(currentBranch) || fail "$(red Failure)" to get current branch from git + local rc; rc=$(prompter 'release candidate sequence number (eg. 1, 2, etc.)' '[0-9]+') + local nextVer; nextVer=$(prompter 'next snapshot version to be released (eg. x.y.z)' '[0-9]+[.][0-9]+[.][0-9]+') + local rcBranch; rcBranch=$ver-rc$rc + local nBranch; nBranch=$rcBranch-next + + cacheGPG || fail "Unable to cache GPG credentials into gpg-agent" + + # create working branch + { + run git branch "$nBranch" "$cBranch" && run git checkout "$nBranch" + } || fail "Unable to create working branch $(red "$nBranch") from $(red "$cBranch")!" + + # create a release candidate from a branch + local oFile; oFile=$(mktemp --tmpdir "fluo-yarn-build-$rcBranch-XXXXXXXX.log") + { + [[ -w $oFile ]] && runLog "$oFile" mvn clean release:clean + } || cleanUpAndFail 'mvn clean release:clean' "$oFile" "$cBranch" "$nBranch" + runLog "$oFile" mvn -B release:prepare -DdevelopmentVersion="${nextVer}-SNAPSHOT" "${extraReleaseArgs}" || \ + cleanUpAndFail "mvn -B release:prepare -DdevelopmentVersion=${nextVer}-SNAPSHOT ${extraReleaseArgs}" "$oFile" "$cBranch" "$nBranch" + runLog "$oFile" mvn release:perform "${extraReleaseArgs}" || \ + cleanUpAndFail "mvn release:perform ${extraReleaseArgs}" "$oFile" "$cBranch" "$nBranch" + + # switch back to original branch + run git checkout "${cBranch}" + + # verify the next branch contains both expected log messages and no more + { + [[ $(gitCommits "${cBranch}..${nBranch}" | wc -l) -eq 2 ]] && \ + [[ $(gitCommit "${nBranch}~2") == $(gitCommit "${cBranch}") ]] && \ + [[ $(gitSubject "${nBranch}") =~ ^\[maven-release-plugin\]\ prepare\ for\ next ]] && \ + [[ $(gitSubject "${nBranch}~1") =~ ^\[maven-release-plugin\]\ prepare\ release\ rel[/] ]] + } || cleanUpAndFail "verifying that $nBranch contains only logs from release plugin" + + # verify the tag is one behind $nBranch and one ahead of $cBranch + [[ $(gitCommit "${nBranch}~1") == $(gitCommit "refs/tags/$tag") ]] || \ + cleanUpAndFail "verifying that ${nBranch}~1 == refs/tags/$tag" + + # remove tag which was created + run git tag -d "$tag" || \ + cleanUpAndFail "removing unused git tag $tag" + + # create release candidate branch to vote on + run git branch "$rcBranch" "${nBranch}~1" || \ + cleanUpAndFail "creating branch $rcBranch" + + # push branches (ask first) + local origin; origin=$(git remote -v | grep ^origin | grep push | awk '{print $2}') + echo "Do you wish to push the following branches to origin ($(green "$origin"))?" + echo " $(yellow "$rcBranch") (for others to examine for the vote)" + echo " $(yellow "$nBranch") (for merging into $cBranch if vote passes)" + local a; a=$(prompter "letter 'y' or 'n'" '[yn]') + { + [[ $a == 'y' ]] && \ + run git push -u origin "refs/heads/$nBranch" "refs/heads/$rcBranch" + } || red "Did not push branches; you'll need to perform this step manually." + + # continue to creating email notification + echo "$(red Running)" "$(yellow "$scriptname" --create-email "$ver" "$rc")" + createEmail "$ver" "$rc" +} + +if [[ $1 == '--create' ]]; then + shift + createReleaseCandidate "$@" +elif [[ $1 == '--email' ]]; then + shift + createEmail "$@" +else + fail "Missing one of: $(red --create), $(red --email)" +fi + diff --git a/core/pom.xml b/core/pom.xml deleted file mode 100644 index 6aa1123..0000000 --- a/core/pom.xml +++ /dev/null @@ -1,60 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more contributor license - agreements. See the NOTICE file distributed with this work for additional information regarding - copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance with the License. You may obtain a - copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software distributed under the License - is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - or implied. See the License for the specific language governing permissions and limitations under - the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.apache.fluo</groupId> - <artifactId>fluo-yarn-project</artifactId> - <version>1.0.0-SNAPSHOT</version> - <relativePath>../pom.xml</relativePath> - </parent> - <artifactId>fluo-yarn-core</artifactId> - <name>Apache Fluo YARN Core</name> - <description>This module contains all code necessary to run Apache Fluo on a YARN cluster using Apache Twill</description> - <dependencies> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-client</artifactId> - <exclusions> - <!-- Excluded so we use asm jar included by twill-yarn. See FLUO-409 --> - <exclusion> - <groupId>asm</groupId> - <artifactId>asm</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.twill</groupId> - <artifactId>twill-api</artifactId> - </dependency> - <dependency> - <groupId>org.apache.twill</groupId> - <artifactId>twill-ext</artifactId> - </dependency> - <dependency> - <groupId>org.apache.twill</groupId> - <artifactId>twill-yarn</artifactId> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </dependency> - </dependencies> -</project> diff --git a/distribution/pom.xml b/distribution/pom.xml deleted file mode 100644 index 68e11e9..0000000 --- a/distribution/pom.xml +++ /dev/null @@ -1,90 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more contributor license - agreements. See the NOTICE file distributed with this work for additional information regarding - copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance with the License. You may obtain a - copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software distributed under the License - is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - or implied. See the License for the specific language governing permissions and limitations under - the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.apache.fluo</groupId> - <artifactId>fluo-yarn-project</artifactId> - <version>1.0.0-SNAPSHOT</version> - <relativePath>../pom.xml</relativePath> - </parent> - <artifactId>fluo-yarn</artifactId> - <packaging>pom</packaging> - <name>Apache Fluo YARN distribution</name> - <description>This module produces a tarball distribution of the YARN launcher for Apache Fluo. It contains all of the - default configuration and scripts required to run Fluo applications in YARN.</description> - <!-- NOTE: These dependencies are bundled in this assembly --> - <dependencies> - <dependency> - <groupId>org.apache.fluo</groupId> - <artifactId>fluo-yarn-core</artifactId> - <optional>true</optional> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <artifactId>maven-assembly-plugin</artifactId> - <executions> - <execution> - <id>bin-assembly</id> - <goals> - <goal>single</goal> - </goals> - <phase>package</phase> - <configuration> - <escapeString>\</escapeString> - <descriptors> - <descriptor>src/main/assembly/bin.xml</descriptor> - </descriptors> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - <profiles> - <profile> - <!-- attach source release when it is created by the apache-release profile --> - <id>apache-release</id> - <build> - <plugins> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> - <executions> - <execution> - <id>attach-source-release-assembly</id> - <goals> - <goal>attach-artifact</goal> - </goals> - <configuration> - <artifacts> - <artifact> - <file>${project.parent.build.directory}/${project.artifactId}-${project.version}-source-release.tar.gz</file> - <type>tar.gz</type> - <classifier>source-release</classifier> - </artifact> - </artifacts> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - </profiles> -</project> diff --git a/pom.xml b/pom.xml index 0547b6f..36c5ba1 100644 --- a/pom.xml +++ b/pom.xml @@ -20,9 +20,9 @@ <artifactId>fluo-parent</artifactId> <version>3</version> </parent> - <artifactId>fluo-yarn-project</artifactId> + <artifactId>fluo-yarn</artifactId> <version>1.0.0-SNAPSHOT</version> - <packaging>pom</packaging> + <packaging>jar</packaging> <name>Apache Fluo YARN launcher</name> <description>Tool that launches Apache Fluo application in Hadoop YARN</description> <licenses> @@ -32,13 +32,18 @@ <distribution>repo</distribution> </license> </licenses> - <modules> - <module>distribution</module> - <module>core</module> - </modules> + <scm> + <connection>scm:git:https://gitbox.apache.org/repos/asf/fluo-yarn.git</connection> + <developerConnection>scm:git:https://gitbox.apache.org/repos/asf/fluo-yarn.git</developerConnection> + <tag>HEAD</tag> + <url>https://gitbox.apache.org/repos/asf?p=fluo-yarn.git</url> + </scm> + <issueManagement> + <system>GitHub</system> + <url>https://github.com/apache/fluo-yarn/issues</url> + </issueManagement> <properties> - <accumulo.version>1.7.3</accumulo.version> - <findbugs.maxRank>9</findbugs.maxRank> + <findbugs.maxRank>12</findbugs.maxRank> <hadoop.version>2.6.3</hadoop.version> <logback.version>1.1.3</logback.version> <slf4j.version>1.7.12</slf4j.version> @@ -79,11 +84,6 @@ <version>4.12</version> </dependency> <dependency> - <groupId>org.apache.fluo</groupId> - <artifactId>fluo-yarn-core</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-client</artifactId> <version>${hadoop.version}</version> @@ -115,52 +115,40 @@ </dependency> </dependencies> </dependencyManagement> + <dependencies> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-client</artifactId> + <exclusions> + <!-- Excluded so we use asm jar included by twill-yarn. See FLUO-409 --> + <exclusion> + <groupId>asm</groupId> + <artifactId>asm</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.twill</groupId> + <artifactId>twill-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.twill</groupId> + <artifactId>twill-ext</artifactId> + </dependency> + <dependency> + <groupId>org.apache.twill</groupId> + <artifactId>twill-yarn</artifactId> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + </dependencies> <build> - <pluginManagement> - <plugins> - <plugin> - <groupId>org.apache.accumulo</groupId> - <artifactId>accumulo-maven-plugin</artifactId> - <version>${accumulo.version}</version> - <configuration> - <instanceName>it-instance-maven</instanceName> - <rootPassword>ITSecret</rootPassword> - </configuration> - <executions> - <execution> - <id>run-plugin</id> - <goals> - <goal>start</goal> - <goal>stop</goal> - </goals> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.rat</groupId> - <artifactId>apache-rat-plugin</artifactId> - <configuration /> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-failsafe-plugin</artifactId> - <configuration> - <systemPropertyVariables> - <fluo.it.instance.name>it-instance-maven</fluo.it.instance.name> - <fluo.it.instance.clear>false</fluo.it.instance.clear> - </systemPropertyVariables> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-release-plugin</artifactId> - <configuration> - <!-- must put this here, due to MRELEASE-963 --> - <tagNameFormat>rel/fluo-yarn-@{project.version}</tagNameFormat> - </configuration> - </plugin> - </plugins> - </pluginManagement> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> @@ -186,39 +174,28 @@ </execution> </executions> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <executions> + <execution> + <id>bin-assembly</id> + <goals> + <goal>single</goal> + </goals> + <phase>package</phase> + <configuration> + <delimiters> + <delimiter>@</delimiter> + </delimiters> + <escapeString>\</escapeString> + <descriptors> + <descriptor>src/main/assembly/bin.xml</descriptor> + </descriptors> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> - <profiles> - <profile> - <id>fluo-release</id> - <!-- some properties to make the release build a bit faster --> - <properties> - <checkstyle.skip>true</checkstyle.skip> - <findbugs.skip>true</findbugs.skip> - <modernizer.skip>true</modernizer.skip> - <skipITs>true</skipITs> - <skipTests>true</skipTests> - </properties> - </profile> - <profile> - <!-- set proper source assembly name with apache-release and don't attach here --> - <id>apache-release</id> - <build> - <pluginManagement> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-assembly-plugin</artifactId> - <inherited>false</inherited> - <configuration> - <!-- source assembly gets attached in the assemble module --> - <attach>false</attach> - <finalName>fluo-${project.version}</finalName> - </configuration> - </plugin> - </plugins> - </pluginManagement> - </build> - </profile> - </profiles> </project> diff --git a/distribution/src/main/assembly/bin.xml b/src/main/assembly/bin.xml similarity index 67% rename from distribution/src/main/assembly/bin.xml rename to src/main/assembly/bin.xml index 68e27b8..4d68291 100644 --- a/distribution/src/main/assembly/bin.xml +++ b/src/main/assembly/bin.xml @@ -21,51 +21,35 @@ <dependencySets> <dependencySet> <outputDirectory>lib</outputDirectory> - <directoryMode>0755</directoryMode> - <fileMode>0644</fileMode> - <useProjectArtifact>false</useProjectArtifact> + <useProjectArtifact>true</useProjectArtifact> <useTransitiveDependencies>false</useTransitiveDependencies> <includes> - <include>${groupId}:fluo-*</include> + <include>${groupId}:${artifactId}:*</include> </includes> </dependencySet> </dependencySets> <fileSets> <fileSet> - <directory>../</directory> + <directory>./</directory> <outputDirectory>.</outputDirectory> <includes> - <include>DISCLAIMER</include> <include>LICENSE</include> <include>NOTICE</include> <include>README.md</include> </includes> </fileSet> <fileSet> - <directory>bin</directory> - <outputDirectory>bin</outputDirectory> - <fileMode>755</fileMode> + <directory>src/main/assembly/contents/</directory> + <outputDirectory>.</outputDirectory> <excludes> - <exclude>bin/fluo-yarn</exclude> + <exclude>**/bin/fluo-yarn</exclude> </excludes> </fileSet> - <fileSet> - <directory>conf</directory> - <outputDirectory>conf</outputDirectory> - </fileSet> - <fileSet> - <directory>lib</directory> - <outputDirectory>lib</outputDirectory> - <fileMode>755</fileMode> - <includes> - <include>fetch.sh</include> - </includes> - </fileSet> </fileSets> <files> <file> - <source>bin/fluo-yarn</source> + <source>src/main/assembly/contents/bin/fluo-yarn</source> <outputDirectory>bin</outputDirectory> <filtered>true</filtered> </file> diff --git a/distribution/bin/fluo-yarn b/src/main/assembly/contents/bin/fluo-yarn similarity index 94% rename from distribution/bin/fluo-yarn rename to src/main/assembly/contents/bin/fluo-yarn index e719380..97e6d1c 100755 --- a/distribution/bin/fluo-yarn +++ b/src/main/assembly/contents/bin/fluo-yarn @@ -28,7 +28,7 @@ export cmd="$1" case "$cmd" in start) export app="$2" ;; esac -export FLUO_YARN_VERSION=${project.version} +export FLUO_YARN_VERSION=@project.version@ if [ ! -f "$conf/fluo-yarn-env.sh" ]; then echo "fluo-yarn-env.sh must exist in $conf" @@ -94,7 +94,7 @@ start) exit 1 fi build_bundled_jar - java org.apache.fluo.yarn.core.FluoYarnLauncher "$FLUO_CONN_PROPS" "$3" "$conf/log4j.properties" "$app" "$app_dir/$bundled_jar" + java org.apache.fluo.yarn.FluoYarnLauncher "$FLUO_CONN_PROPS" "$3" "$conf/log4j.properties" "$app" "$app_dir/$bundled_jar" ;; classpath) echo "$CLASSPATH" diff --git a/distribution/conf/fluo-yarn-env.sh b/src/main/assembly/contents/conf/fluo-yarn-env.sh similarity index 100% rename from distribution/conf/fluo-yarn-env.sh rename to src/main/assembly/contents/conf/fluo-yarn-env.sh diff --git a/distribution/conf/fluo-yarn.properties b/src/main/assembly/contents/conf/fluo-yarn.properties similarity index 100% rename from distribution/conf/fluo-yarn.properties rename to src/main/assembly/contents/conf/fluo-yarn.properties diff --git a/distribution/conf/log4j.properties b/src/main/assembly/contents/conf/log4j.properties similarity index 100% rename from distribution/conf/log4j.properties rename to src/main/assembly/contents/conf/log4j.properties diff --git a/distribution/conf/logback.xml b/src/main/assembly/contents/conf/logback.xml similarity index 100% rename from distribution/conf/logback.xml rename to src/main/assembly/contents/conf/logback.xml diff --git a/distribution/lib/fetch.sh b/src/main/assembly/contents/lib/fetch.sh similarity index 100% rename from distribution/lib/fetch.sh rename to src/main/assembly/contents/lib/fetch.sh diff --git a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java b/src/main/java/org/apache/fluo/yarn/FluoYarnEnv.java similarity index 99% rename from core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java rename to src/main/java/org/apache/fluo/yarn/FluoYarnEnv.java index 15e77b0..03a02c6 100644 --- a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java +++ b/src/main/java/org/apache/fluo/yarn/FluoYarnEnv.java @@ -13,7 +13,7 @@ * the License. */ -package org.apache.fluo.yarn.core; +package org.apache.fluo.yarn; import java.io.File; import java.nio.file.Paths; diff --git a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnLauncher.java b/src/main/java/org/apache/fluo/yarn/FluoYarnLauncher.java similarity index 99% rename from core/src/main/java/org/apache/fluo/yarn/core/FluoYarnLauncher.java rename to src/main/java/org/apache/fluo/yarn/FluoYarnLauncher.java index 056604f..ea50683 100644 --- a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnLauncher.java +++ b/src/main/java/org/apache/fluo/yarn/FluoYarnLauncher.java @@ -13,7 +13,7 @@ * the License. */ -package org.apache.fluo.yarn.core; +package org.apache.fluo.yarn; import java.io.File; import java.util.Collection; diff --git a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java b/src/main/java/org/apache/fluo/yarn/FluoYarnProperties.java similarity index 98% rename from core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java rename to src/main/java/org/apache/fluo/yarn/FluoYarnProperties.java index c707928..6a0d31e 100644 --- a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java +++ b/src/main/java/org/apache/fluo/yarn/FluoYarnProperties.java @@ -13,7 +13,7 @@ * the License. */ -package org.apache.fluo.yarn.core; +package org.apache.fluo.yarn; import java.io.FileInputStream; import java.io.IOException; -- To stop receiving notification emails like this one, please contact ctubb...@apache.org.