XComp commented on code in PR #24465:
URL: https://github.com/apache/flink/pull/24465#discussion_r1543160518
##########
flink-end-to-end-tests/test-scripts/test_file_sink.sh:
##########
@@ -79,30 +42,69 @@
TEST_PROGRAM_JAR="${END_TO_END_DIR}/flink-file-sink-test/target/FileSinkProgram.
# sorted content of part files
###################################
function get_complete_result {
- if [ "${OUT_TYPE}" == "s3" ]; then
- s3_get_by_full_path_and_filename_prefix "$OUTPUT_PATH" "$S3_PREFIX"
"part-" true
- fi
find "${OUTPUT_PATH}" -type f \( -iname "part-*" \) -exec cat {} + | sort -g
}
###################################
# Get total number of lines in part files.
#
# Globals:
-# S3_PREFIX
+# OUTPUT_PATH
# Arguments:
# None
# Returns:
# line number in part files
###################################
function get_total_number_of_valid_lines {
- if [ "${OUT_TYPE}" == "local" ]; then
- get_complete_result | wc -l | tr -d '[:space:]'
- elif [ "${OUT_TYPE}" == "s3" ]; then
- s3_get_number_of_lines_by_prefix "${S3_PREFIX}" "part-"
- fi
+ get_complete_result | wc -l | tr -d '[:space:]'
}
+if [ "${OUT_TYPE}" == "local" ]; then
+ echo "[INFO] Test run in local environment: No S3 environment is not loaded."
+elif [ "${OUT_TYPE}" == "s3" ]; then
+ source "$(dirname "$0")"/common_s3_minio.sh
+ s3_setup hadoop
+
+ # overwrites JOB_OUTPUT_PATH to point to S3
+ S3_DATA_PREFIX="${RANDOM_PREFIX}"
+ S3_CHECKPOINT_PREFIX="${RANDOM_PREFIX}-chk"
+ JOB_OUTPUT_PATH="s3://$IT_CASE_S3_BUCKET/${S3_DATA_PREFIX}"
+ set_config_key "state.checkpoints.dir"
"s3://$IT_CASE_S3_BUCKET/${S3_CHECKPOINT_PREFIX}"
Review Comment:
The folder was not used anywhere in the old version. That is why I dropped
this call.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@flink.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
