This is an automated email from the ASF dual-hosted git repository.
mapohl pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new f29ee07f5e5 [FLINK-34047][ci] Make CI scripts agnostic to CI
environment (#24061)
f29ee07f5e5 is described below
commit f29ee07f5e568fa94da75f3879a9aa7016dfcd5a
Author: Matthias Pohl <matthias.p...@aiven.io>
AuthorDate: Fri Jan 12 08:46:25 2024 +0100
[FLINK-34047][ci] Make CI scripts agnostic to CI environment (#24061)
- Makes prepare_debug_files not rely on "magic" environment variables
anymore
- Makes uploading_watchdog.sh support Azure Pipelines and GitHub Actions
---
tools/azure-pipelines/debug_files_utils.sh | 14 +++++++++---
tools/azure-pipelines/uploading_watchdog.sh | 33 +++++++++++++++++++++++++++--
2 files changed, 42 insertions(+), 5 deletions(-)
diff --git a/tools/azure-pipelines/debug_files_utils.sh
b/tools/azure-pipelines/debug_files_utils.sh
index 50c1b4c5a4d..35c51724158 100755
--- a/tools/azure-pipelines/debug_files_utils.sh
+++ b/tools/azure-pipelines/debug_files_utils.sh
@@ -18,9 +18,17 @@
################################################################################
function prepare_debug_files {
- MODULE=$@
- export DEBUG_FILES_OUTPUT_DIR="$AGENT_TEMPDIRECTORY/debug_files"
- export DEBUG_FILES_NAME="$(echo $MODULE | tr -c '[:alnum:]\n\r'
'_')-$(date +%s)"
+ if [ "$#" != "2" ]; then
+ echo "[ERROR] Invalid number of parameters passed. Expected
parameters for $0: <parent-directory> <module-label>"
+ exit 1
+ fi
+
+ local parent_directory module
+ parent_directory="$1"
+ module="$2"
+
+ export DEBUG_FILES_OUTPUT_DIR="${parent_directory}/debug_files"
+ export DEBUG_FILES_NAME="$(echo "${module}" | tr -c '[:alnum:]\n\r'
'_')-$(date +%s)"
echo "##vso[task.setvariable
variable=DEBUG_FILES_OUTPUT_DIR]$DEBUG_FILES_OUTPUT_DIR"
echo "##vso[task.setvariable
variable=DEBUG_FILES_NAME]$DEBUG_FILES_NAME"
mkdir -p $DEBUG_FILES_OUTPUT_DIR || { echo "FAILURE: cannot create
debug files directory '${DEBUG_FILES_OUTPUT_DIR}'." ; exit 1; }
diff --git a/tools/azure-pipelines/uploading_watchdog.sh
b/tools/azure-pipelines/uploading_watchdog.sh
index 152f11581d0..2eafc1dfcbe 100755
--- a/tools/azure-pipelines/uploading_watchdog.sh
+++ b/tools/azure-pipelines/uploading_watchdog.sh
@@ -27,16 +27,45 @@ if [ -z "$HERE" ] ; then
exit 1
fi
+if [ -n "${TF_BUILD+x}" ]; then
+ echo "[INFO] Azure Pipelines environment detected: $0 will rely on
Azure-specific environment variables."
+ job_name="$AGENT_JOBNAME"
+ temporary_folder="$AGENT_TEMPDIRECTORY"
+ job_timeout="$SYSTEM_JOBTIMEOUT"
+ pipeline_start_time="${SYSTEM_PIPELINESTARTTIME}"
+elif [ -n "${GITHUB_ACTIONS+x}" ]; then
+ echo "[INFO] GitHub Actions environment detected: $0 will rely on
GHA-specific environment variables."
+ job_name="${GITHUB_JOB}"
+ temporary_folder="${RUNNER_TEMP}"
+
+ if [ -n "${GHA_JOB_TIMEOUT+x}" ]; then
+ job_timeout="${GHA_JOB_TIMEOUT}"
+ else
+ echo "[ERROR] GHA_JOB_TIMEOUT is not set: GHA_JOB_TIMEOUT needs to be set
by the workflow because there is no pre-defined environment variable available
for this parameter for now."
+ exit 1
+ fi
+
+ if [ -n "${GHA_PIPELINE_START_TIME+x}" ]; then
+ pipeline_start_time="${GHA_PIPELINE_START_TIME}"
+ else
+ echo "[ERROR] GHA_PIPELINE_START_TIME is not set: GHA_PIPELINE_START_TIME
needs to be set by the workflow because there is no pre-defined environment
variable available for this parameter for now."
+ exit 1
+ fi
+else
+ echo "[ERROR] No CI environment detected that could be used for injecting
the parameters which are needed by $0."
+ exit 1
+fi
+
source "${HERE}/../ci/controller_utils.sh"
source ./tools/azure-pipelines/debug_files_utils.sh
-prepare_debug_files "$AGENT_JOBNAME"
+prepare_debug_files "${temporary_folder}" "${job_name}"
export FLINK_LOG_DIR="$DEBUG_FILES_OUTPUT_DIR/flink-logs"
mkdir $FLINK_LOG_DIR || { echo "FAILURE: cannot create log directory
'${FLINK_LOG_DIR}'." ; exit 1; }
sudo apt-get install -y moreutils
REAL_START_SECONDS=$(date +"%s")
-REAL_END_SECONDS=$(date -d "$SYSTEM_PIPELINESTARTTIME + $SYSTEM_JOBTIMEOUT
minutes" +"%s")
+REAL_END_SECONDS=$(date -d "${pipeline_start_time} + ${job_timeout} minutes"
+"%s")
REAL_TIMEOUT_SECONDS=$(($REAL_END_SECONDS - $REAL_START_SECONDS))
KILL_SECONDS_BEFORE_TIMEOUT=$((2 * 60))
