RyanSkraba commented on code in PR #24465:
URL: https://github.com/apache/flink/pull/24465#discussion_r1541478084
##########
flink-end-to-end-tests/test-scripts/test_file_sink.sh:
##########
@@ -79,30 +42,69 @@
TEST_PROGRAM_JAR="${END_TO_END_DIR}/flink-file-sink-test/target/FileSinkProgram.
# sorted content of part files
###################################
function get_complete_result {
- if [ "${OUT_TYPE}" == "s3" ]; then
- s3_get_by_full_path_and_filename_prefix "$OUTPUT_PATH" "$S3_PREFIX"
"part-" true
- fi
find "${OUTPUT_PATH}" -type f \( -iname "part-*" \) -exec cat {} + | sort -g
}
###################################
# Get total number of lines in part files.
#
# Globals:
-# S3_PREFIX
+# OUTPUT_PATH
# Arguments:
# None
# Returns:
# line number in part files
###################################
function get_total_number_of_valid_lines {
- if [ "${OUT_TYPE}" == "local" ]; then
- get_complete_result | wc -l | tr -d '[:space:]'
- elif [ "${OUT_TYPE}" == "s3" ]; then
- s3_get_number_of_lines_by_prefix "${S3_PREFIX}" "part-"
- fi
+ get_complete_result | wc -l | tr -d '[:space:]'
}
+if [ "${OUT_TYPE}" == "local" ]; then
+ echo "[INFO] Test run in local environment: No S3 environment is not loaded."
+elif [ "${OUT_TYPE}" == "s3" ]; then
+ source "$(dirname "$0")"/common_s3_minio.sh
+ s3_setup hadoop
+
+ # overwrites JOB_OUTPUT_PATH to point to S3
+ S3_DATA_PREFIX="${RANDOM_PREFIX}"
+ S3_CHECKPOINT_PREFIX="${RANDOM_PREFIX}-chk"
+ JOB_OUTPUT_PATH="s3://$IT_CASE_S3_BUCKET/${S3_DATA_PREFIX}"
+ set_config_key "state.checkpoints.dir"
"s3://$IT_CASE_S3_BUCKET/${S3_CHECKPOINT_PREFIX}"
Review Comment:
In the original, there was a line `mkdir -p "$OUTPUT_PATH-chk"`. Is this no
longer necessary?
##########
flink-end-to-end-tests/test-scripts/test_file_sink.sh:
##########
@@ -79,30 +42,69 @@
TEST_PROGRAM_JAR="${END_TO_END_DIR}/flink-file-sink-test/target/FileSinkProgram.
# sorted content of part files
###################################
function get_complete_result {
- if [ "${OUT_TYPE}" == "s3" ]; then
- s3_get_by_full_path_and_filename_prefix "$OUTPUT_PATH" "$S3_PREFIX"
"part-" true
- fi
find "${OUTPUT_PATH}" -type f \( -iname "part-*" \) -exec cat {} + | sort -g
}
###################################
# Get total number of lines in part files.
#
# Globals:
-# S3_PREFIX
+# OUTPUT_PATH
# Arguments:
# None
# Returns:
# line number in part files
###################################
function get_total_number_of_valid_lines {
- if [ "${OUT_TYPE}" == "local" ]; then
- get_complete_result | wc -l | tr -d '[:space:]'
- elif [ "${OUT_TYPE}" == "s3" ]; then
- s3_get_number_of_lines_by_prefix "${S3_PREFIX}" "part-"
- fi
+ get_complete_result | wc -l | tr -d '[:space:]'
}
+if [ "${OUT_TYPE}" == "local" ]; then
+ echo "[INFO] Test run in local environment: No S3 environment is not loaded."
+elif [ "${OUT_TYPE}" == "s3" ]; then
+ source "$(dirname "$0")"/common_s3_minio.sh
+ s3_setup hadoop
+
+ # overwrites JOB_OUTPUT_PATH to point to S3
+ S3_DATA_PREFIX="${RANDOM_PREFIX}"
+ S3_CHECKPOINT_PREFIX="${RANDOM_PREFIX}-chk"
+ JOB_OUTPUT_PATH="s3://$IT_CASE_S3_BUCKET/${S3_DATA_PREFIX}"
+ set_config_key "state.checkpoints.dir"
"s3://$IT_CASE_S3_BUCKET/${S3_CHECKPOINT_PREFIX}"
+
+ # overwrites implementation for local runs
+ function get_complete_result {
+ # copies the data from S3 to the local OUTPUT_PATH
+ s3_get_by_full_path_and_filename_prefix "$OUTPUT_PATH"
"$FILE_SINK_TEST_TEMP_SUBFOLDER" "part-" true
+
+ # and prints the sorted output
+ find "${OUTPUT_PATH}" -type f \( -iname "part-*" \) -exec cat {} + | sort
-g
+ }
+
+ # overwrites implementation for local runs
+ function get_total_number_of_valid_lines {
+ s3_get_number_of_lines_by_prefix "${FILE_SINK_TEST_TEMP_SUBFOLDER}" "part-"
+ }
+
+ # make sure we delete the file at the end
+ function out_cleanup {
+ s3_delete_by_full_path_prefix "${S3_DATA_PREFIX}"
+ s3_delete_by_full_path_prefix "${S3_CHECKPOINT_PREFIX}"
+ }
+
+ on_exit out_cleanup
+else
+ echo "[ERROR] Unknown out type: ${OUT_TYPE}"
+ exit 1
+fi
+
+# randomly set up openSSL with dynamically/statically linked libraries
Review Comment:
Did you mean to copy the `OPENSSL_LINKAGE` lines here?
##########
flink-end-to-end-tests/test-scripts/test_file_sink.sh:
##########
@@ -79,30 +42,69 @@
TEST_PROGRAM_JAR="${END_TO_END_DIR}/flink-file-sink-test/target/FileSinkProgram.
# sorted content of part files
###################################
function get_complete_result {
- if [ "${OUT_TYPE}" == "s3" ]; then
- s3_get_by_full_path_and_filename_prefix "$OUTPUT_PATH" "$S3_PREFIX"
"part-" true
- fi
find "${OUTPUT_PATH}" -type f \( -iname "part-*" \) -exec cat {} + | sort -g
}
###################################
# Get total number of lines in part files.
#
# Globals:
-# S3_PREFIX
+# OUTPUT_PATH
# Arguments:
# None
# Returns:
# line number in part files
###################################
function get_total_number_of_valid_lines {
- if [ "${OUT_TYPE}" == "local" ]; then
- get_complete_result | wc -l | tr -d '[:space:]'
- elif [ "${OUT_TYPE}" == "s3" ]; then
- s3_get_number_of_lines_by_prefix "${S3_PREFIX}" "part-"
- fi
+ get_complete_result | wc -l | tr -d '[:space:]'
}
+if [ "${OUT_TYPE}" == "local" ]; then
+ echo "[INFO] Test run in local environment: No S3 environment is not loaded."
Review Comment:
```suggestion
echo "[INFO] Test run in local environment: No S3 environment is loaded."
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@flink.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
