[
https://issues.apache.org/jira/browse/FLINK-8819?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16626305#comment-16626305
]
ASF GitHub Bot commented on FLINK-8819:
---------------------------------------
GJL commented on a change in pull request #6642: [FLINK-8819][travis] Rework
travis script to use stages
URL: https://github.com/apache/flink/pull/6642#discussion_r219956271
##########
File path: tools/travis_controller.sh
##########
@@ -0,0 +1,211 @@
+#!/usr/bin/env bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+CACHE_DIR="$HOME/flink_cache"
+CACHE_BUILD_DIR="$CACHE_DIR/$TRAVIS_BUILD_NUMBER"
+CACHE_FLINK_DIR="$CACHE_BUILD_DIR/flink"
+
+HERE="`dirname \"$0\"`" # relative
+HERE="`( cd \"$HERE\" && pwd )`" # absolutized and normalized
+if [ -z "$HERE" ] ; then
+ # error; for some reason, the path is not accessible
+ # to the script (e.g. permissions re-evaled after suid)
+ exit 1 # fail
+fi
+
+source "${HERE}/travis/fold.sh"
+source "${HERE}/travis/stage.sh"
+source "${HERE}/travis/shade.sh"
+
+function deleteOldCaches() {
+ while read CACHE_DIR; do
+ local old_number="${CACHE_DIR##*/}"
+ if [ "$old_number" -lt "$TRAVIS_BUILD_NUMBER" ]; then
+ echo "Deleting old cache $CACHE_DIR"
+ rm -rf "$CACHE_DIR"
+ fi
+ done
+}
+
+# delete leftover caches from previous builds
+find "$CACHE_DIR" -mindepth 1 -maxdepth 1 | grep -v "$TRAVIS_BUILD_NUMBER" |
deleteOldCaches
+
+function getCurrentStage() {
+ STAGE_NUMBER=$(echo "$TRAVIS_JOB_NUMBER" | cut -d'.' -f 2)
+ case $STAGE_NUMBER in
+ (1)
+ echo "$STAGE_COMPILE"
+ ;;
+ (2)
+ echo "$STAGE_COMPILE"
+ ;;
+ (3)
+ echo "$STAGE_CORE"
+ ;;
+ (4)
+ echo "$STAGE_LIBRARIES"
+ ;;
+ (5)
+ echo "$STAGE_CONNECTORS"
+ ;;
+ (6)
+ echo "$STAGE_TESTS"
+ ;;
+ (7)
+ echo "$STAGE_MISC"
+ ;;
+ (8)
+ echo "$STAGE_CORE"
+ ;;
+ (9)
+ echo "$STAGE_LIBRARIES"
+ ;;
+ (10)
+ echo "$STAGE_CONNECTORS"
+ ;;
+ (11)
+ echo "$STAGE_TESTS"
+ ;;
+ (12)
+ echo "$STAGE_MISC"
+ ;;
+ (13)
+ echo "$STAGE_CLEANUP"
+ ;;
+ (14)
+ echo "$STAGE_CLEANUP"
+ ;;
+ (*)
+ echo "Invalid stage detected ($STAGE_NUMBER)"
+ return 1
+ ;;
+ esac
+
+ return 0
+}
+
+STAGE=$(getCurrentStage)
+if [ $? != 0 ]; then
+ echo "Could not determine current stage."
+ exit 1
+fi
+echo "Current stage: \"$STAGE\""
+
+EXIT_CODE=0
+
+# Run actual compile&test steps
+if [ $STAGE == "$STAGE_COMPILE" ]; then
+ MVN="mvn clean install -nsu -Dflink.forkCount=2
-Dflink.forkCountTestPackage=2 -Dmaven.javadoc.skip=true -B -DskipTests
$PROFILE"
+ $MVN
+ EXIT_CODE=$?
+
+ if [ $EXIT_CODE == 0 ]; then
+ printf
"\n\n==============================================================================\n"
+ printf "Checking dependency convergence\n"
+ printf
"==============================================================================\n"
+
+ ./tools/check_dependency_convergence.sh
+ EXIT_CODE=$?
+ else
+ printf
"\n==============================================================================\n"
+ printf "Previous build failure detected, skipping
dependency-convergence check.\n"
+ printf
"==============================================================================\n"
+ fi
+
+ if [ $EXIT_CODE == 0 ]; then
+ check_shaded_artifacts
+ EXIT_CODE=$(($EXIT_CODE+$?))
+ check_shaded_artifacts_s3_fs hadoop
+ EXIT_CODE=$(($EXIT_CODE+$?))
+ check_shaded_artifacts_s3_fs presto
+ EXIT_CODE=$(($EXIT_CODE+$?))
+ check_shaded_artifacts_connector_elasticsearch ""
+ EXIT_CODE=$(($EXIT_CODE+$?))
+ check_shaded_artifacts_connector_elasticsearch 2
+ EXIT_CODE=$(($EXIT_CODE+$?))
+ check_shaded_artifacts_connector_elasticsearch 5
+ EXIT_CODE=$(($EXIT_CODE+$?))
+ else
+ echo
"=============================================================================="
+ echo "Previous build failure detected, skipping shaded dependency
check."
+ echo
"=============================================================================="
+ fi
+
+ if [ $EXIT_CODE == 0 ]; then
+ echo "Creating cache build directory $CACHE_FLINK_DIR"
+ mkdir -p "$CACHE_FLINK_DIR"
+
+ cp -r . "$CACHE_FLINK_DIR"
+
+ function minimizeCachedFiles() {
+ # reduces the size of the cached directory to speed up
+ # the packing&upload / download&unpacking process
+ # by removing files not required for subsequent stages
+
+ # original jars
+ find "$CACHE_FLINK_DIR" -maxdepth 8 -type f -name 'original-*.jar'
| xargs rm -rf
+
+ # .git directory
+ # not deleting this can cause build stability issues
+ # merging the cached version sometimes fails
+ rm -rf "$CACHE_FLINK_DIR/.git"
+ }
+
+ start_fold "minimize_cache" "Minimizing cache"
+ travis_time_start
+ minimizeCachedFiles
+ travis_time_finish
+ end_fold "minimize_cache"
+ else
+ echo
"=============================================================================="
+ echo "Previous build failure detected, skipping cache setup."
+ echo
"=============================================================================="
+ fi
+elif [ $STAGE != "$STAGE_CLEANUP" ]; then
+ if ! [ -e $CACHE_FLINK_DIR ]; then
+ echo "Cached flink dir $CACHE_FLINK_DIR does not exist. Exiting
build."
+ exit 1
+ fi
+ # merged compiled flink into local clone
+ # this prevents the cache from being re-uploaded
+ start_fold "merge_cache" "Merging cache"
+ travis_time_start
+ cp -RT "$CACHE_FLINK_DIR" "."
+ travis_time_finish
+ end_fold "merge_cache"
+
+ start_fold "adjust_timestamps" "Adjusting timestamps"
+ travis_time_start
+ # adjust timestamps to prevent recompilation
+ find . -type f -name '*.java' | xargs touch
Review comment:
By `touch`ing all files, we set the mod time of all source files to _"now"_.
How is this older than the compiled files? It seems to work, though.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Rework travis script to use build stages
> ----------------------------------------
>
> Key: FLINK-8819
> URL: https://issues.apache.org/jira/browse/FLINK-8819
> Project: Flink
> Issue Type: Sub-task
> Components: Build System, Travis
> Reporter: Chesnay Schepler
> Assignee: Chesnay Schepler
> Priority: Trivial
> Labels: pull-request-available
>
> This issue is for tracking efforts to rework our Travis scripts to use
> [stages|https://docs.travis-ci.com/user/build-stages/].
> This feature allows us to define a sequence of jobs that are run one after
> another. This implies that we can define dependencies between jobs, in
> contrast to our existing jobs that have to be self-contained.
> As an example, we could have a compile stage, and a test stage with multiple
> jobs.
> The main benefit here is that we no longer have to compile modules multiple
> times, which would reduce our build times.
> The major issue here however is that there is no _proper_ support for passing
> build-artifacts from one stage to the next. According to this
> [issue|https://github.com/travis-ci/beta-features/issues/28] it is on their
> to-do-list however.
> In the mean-time we could manually transfer the artifacts between stages by
> either using the Travis cache or some other external storage. The cache
> solution would work by setting up a cached directory (just like the mvn
> cache) and creating build-scope directories within containing the artifacts
> (I have a prototype that works like this).
> The major concern here is that of cleaning up the cache/storage.
> We can clean things up if
> * our script fails
> * the last stage succeeds.
> We can *not* clean things up if
> * the build is canceled
> * travis fails the build due to a timeout or similar
> as apparently there is [no way to run a script at the end of a
> build|https://github.com/travis-ci/travis-ci/issues/4221].
> Thus we would either have to periodically clear the cache, or encode more
> information into the cached files that would allow _other_ builds to clean up
> stale date. (For example the build number or date).
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
