github-actions[bot] commented on PR #21788:
URL: https://github.com/apache/doris/pull/21788#issuecomment-1637535465
#### `sh-checker report`
To get the full details, please check in the
[job]("https://github.com/apache/doris/actions/runs/5573038925";) output.
<details>
<summary>shellcheck errors</summary>
```
'shellcheck ' returned error 1 finding the following syntactical issues:
----------
In tools/emr_storage_regression/emr_tools.sh line 1:
set -eo pipefail
^-- SC2148 (error): Tips depend on target shell and yours is unknown. Add a
shebang or a 'shell' directive.
In tools/emr_storage_regression/emr_tools.sh line 94:
shift
^---^ SC2317 (info): Command appears to be unreachable. Check usage
(or ignore if invoked indirectly).
In tools/emr_storage_regression/ping_test/ping_poc.sh line 12:
FE_HOST=${FE_HOST}
^-----^ SC2269 (info): This variable is assigned to itself, so the
assignment does nothing.
In tools/emr_storage_regression/ping_test/ping_poc.sh line 13:
FE_QUERY_PORT=${FE_QUERY_PORT}
^-----------^ SC2269 (info): This variable is assigned to itself, so the
assignment does nothing.
In tools/emr_storage_regression/ping_test/ping_poc.sh line 14:
USER=${USER}
^--^ SC2269 (info): This variable is assigned to itself, so the assignment
does nothing.
In tools/emr_storage_regression/ping_test/ping_poc.sh line 117:
for c in $(echo "${EMR_CATALOG}"); do
^----------------------^ SC2116 (style): Useless echo? Instead of
'cmd $(echo foo)', just use 'cmd foo'.
In tools/emr_storage_regression/standard_set/gen_spark_create_sql.sh line 6:
sh gen_tbl/gen_ssb_create_sql.sh "${BUCKET}"/ssb/ssb100_parquet
ssb100_parquet_"${TYPE}" parquet >> create_"${TYPE}".sql
^-- SC2129 (style): Consider using { cmd1; cmd2; } >> file instead of
individual redirects.
In
tools/emr_storage_regression/standard_set/gen_tbl/gen_clickbench_create_sql.sh
line 2:
if [ -z $1 ]; then
^-------^ SC2292 (style): Prefer [[ ]] over [ ] for tests in Bash/Ksh.
^-- SC2086 (info): Double quote to prevent globbing and word
splitting.
Did you mean:
if [[ -z "$1" ]]; then
In
tools/emr_storage_regression/standard_set/gen_tbl/gen_clickbench_create_sql.sh
line 9:
if [ -z $2 ]; then
^-------^ SC2292 (style): Prefer [[ ]] over [ ] for tests in Bash/Ksh.
^-- SC2086 (info): Double quote to prevent globbing and word
splitting.
Did you mean:
if [[ -z "$2" ]]; then
In
tools/emr_storage_regression/standard_set/gen_tbl/gen_clickbench_create_sql.sh
line 15:
if [ -z $3 ]; then
^-------^ SC2292 (style): Prefer [[ ]] over [ ] for tests in Bash/Ksh.
^-- SC2086 (info): Double quote to prevent globbing and word
splitting.
Did you mean:
if [[ -z "$3" ]]; then
In
tools/emr_storage_regression/standard_set/gen_tbl/gen_clickbench_create_sql.sh
line 23:
USE '"${db}"';
^-- SC2016 (info): Expressions don't expand in single quotes,
use double quotes for that.
In tools/emr_storage_regression/standard_set/gen_tbl/gen_ssb_create_sql.sh
line 2:
if [ -z $1 ]; then
^-------^ SC2292 (style): Prefer [[ ]] over [ ] for tests in Bash/Ksh.
^-- SC2086 (info): Double quote to prevent globbing and word
splitting.
Did you mean:
if [[ -z "$1" ]]; then
In tools/emr_storage_regression/standard_set/gen_tbl/gen_ssb_create_sql.sh
line 9:
if [ -z $2 ]; then
^-------^ SC2292 (style): Prefer [[ ]] over [ ] for tests in Bash/Ksh.
^-- SC2086 (info): Double quote to prevent globbing and word
splitting.
Did you mean:
if [[ -z "$2" ]]; then
In tools/emr_storage_regression/standard_set/gen_tbl/gen_ssb_create_sql.sh
line 16:
if [ -z $3 ]; then
^-------^ SC2292 (style): Prefer [[ ]] over [ ] for tests in Bash/Ksh.
^-- SC2086 (info): Double quote to prevent globbing and word
splitting.
Did you mean:
if [[ -z "$3" ]]; then
In tools/emr_storage_regression/standard_set/gen_tbl/gen_ssb_create_sql.sh
line 24:
USE '"${db}"';
^-- SC2016 (info): Expressions don't expand in single quotes,
use double quotes for that.
In tools/emr_storage_regression/standard_set/gen_tbl/gen_ssb_create_sql.sh
line 36:
LOCATION "'"${db_loc}"/customer'";
^-- SC2016 (info): Expressions don't expand
in single quotes, use double quotes for that.
In tools/emr_storage_regression/standard_set/gen_tbl/gen_ssb_create_sql.sh
line 57:
LOCATION "'"${db_loc}"/dates'";
^-- SC2016 (info): Expressions don't expand in
single quotes, use double quotes for that.
In tools/emr_storage_regression/standard_set/gen_tbl/gen_ssb_create_sql.sh
line 78:
LOCATION "'"${db_loc}"/lineorder'";
^-- SC2016 (info): Expressions don't expand
in single quotes, use double quotes for that.
In tools/emr_storage_regression/standard_set/gen_tbl/gen_ssb_create_sql.sh
line 91:
LOCATION "'"${db_loc}"/part'";
^-- SC2016 (info): Expressions don't expand in
single quotes, use double quotes for that.
In tools/emr_storage_regression/standard_set/gen_tbl/gen_ssb_create_sql.sh
line 102:
LOCATION "'"${db_loc}"/supplier'";
^-- SC2016 (info): Expressions don't expand
in single quotes, use double quotes for that.
In tools/emr_storage_regression/standard_set/gen_tbl/gen_tpch_create_sql.sh
line 2:
if [ -z $1 ]; then
^-------^ SC2292 (style): Prefer [[ ]] over [ ] for tests in Bash/Ksh.
^-- SC2086 (info): Double quote to prevent globbing and word
splitting.
Did you mean:
if [[ -z "$1" ]]; then
In tools/emr_storage_regression/standard_set/gen_tbl/gen_tpch_create_sql.sh
line 9:
if [ -z $2 ]; then
^-------^ SC2292 (style): Prefer [[ ]] over [ ] for tests in Bash/Ksh.
^-- SC2086 (info): Double quote to prevent globbing and word
splitting.
Did you mean:
if [[ -z "$2" ]]; then
In tools/emr_storage_regression/standard_set/gen_tbl/gen_tpch_create_sql.sh
line 16:
if [ -z $3 ]; then
^-------^ SC2292 (style): Prefer [[ ]] over [ ] for tests in Bash/Ksh.
^-- SC2086 (info): Double quote to prevent globbing and word
splitting.
Did you mean:
if [[ -z "$3" ]]; then
In tools/emr_storage_regression/standard_set/gen_tbl/gen_tpch_create_sql.sh
line 24:
USE '"${db}"';
^-- SC2016 (info): Expressions don't expand in single quotes,
use double quotes for that.
In tools/emr_storage_regression/standard_set/gen_tbl/gen_tpch_create_sql.sh
line 36:
LOCATION "'"${db_loc}"/customer'";
^-- SC2016 (info): Expressions don't expand
in single quotes, use double quotes for that.
In tools/emr_storage_regression/standard_set/gen_tbl/gen_tpch_create_sql.sh
line 56:
LOCATION "'"${db_loc}"/lineitem'";
^-- SC2016 (info): Expressions don't expand
in single quotes, use double quotes for that.
In tools/emr_storage_regression/standard_set/gen_tbl/gen_tpch_create_sql.sh
line 64:
LOCATION "'"${db_loc}"/nation'";
^-- SC2016 (info): Expressions don't expand in
single quotes, use double quotes for that.
In tools/emr_storage_regression/standard_set/gen_tbl/gen_tpch_create_sql.sh
line 77:
LOCATION "'"${db_loc}"/orders'";
^-- SC2016 (info): Expressions don't expand in
single quotes, use double quotes for that.
In tools/emr_storage_regression/standard_set/gen_tbl/gen_tpch_create_sql.sh
line 90:
LOCATION "'"${db_loc}"/part'";
^-- SC2016 (info): Expressions don't expand in
single quotes, use double quotes for that.
In tools/emr_storage_regression/standard_set/gen_tbl/gen_tpch_create_sql.sh
line 99:
LOCATION "'"${db_loc}"/partsupp'";
^-- SC2016 (info): Expressions don't expand
in single quotes, use double quotes for that.
In tools/emr_storage_regression/standard_set/gen_tbl/gen_tpch_create_sql.sh
line 106:
LOCATION "'"${db_loc}"/region'";
^-- SC2016 (info): Expressions don't expand in
single quotes, use double quotes for that.
In tools/emr_storage_regression/standard_set/run_queries.sh line 9:
echo $DB
^-^ SC2086 (info): Double quote to prevent globbing and word splitting.
^-^ SC2250 (style): Prefer putting braces around variable references
even when not strictly required.
Did you mean:
echo "${DB}"
In tools/emr_storage_regression/standard_set/run_queries.sh line 14:
touch $RESULT_FILE
^----------^ SC2086 (info): Double quote to prevent globbing and word
splitting.
^----------^ SC2250 (style): Prefer putting braces around variable
references even when not strictly required.
Did you mean:
touch "${RESULT_FILE}"
In tools/emr_storage_regression/standard_set/run_queries.sh line 15:
truncate -s 0 $RESULT_FILE
^----------^ SC2086 (info): Double quote to prevent globbing
and word splitting.
^----------^ SC2250 (style): Prefer putting braces around
variable references even when not strictly required.
Did you mean:
truncate -s 0 "${RESULT_FILE}"
In tools/emr_storage_regression/standard_set/run_queries.sh line 18:
echo -n "query${QUERY_NUM}," | tee -a $RESULT_FILE
^----------^ SC2086 (info): Double
quote to prevent globbing and word splitting.
^----------^ SC2250 (style):
Prefer putting braces around variable references even when not strictly
required.
Did you mean:
echo -n "query${QUERY_NUM}," | tee -a "${RESULT_FILE}"
In tools/emr_storage_regression/standard_set/run_queries.sh line 19:
for i in $(seq 1 $TRIES); do
^----^ SC2248 (style): Prefer double quoting even when
variables don't contain special characters.
^----^ SC2250 (style): Prefer putting braces around
variable references even when not strictly required.
Did you mean:
for i in $(seq 1 "${TRIES}"); do
In tools/emr_storage_regression/standard_set/run_queries.sh line 21:
echo -n "$RES" | tee -a $RESULT_FILE
^--^ SC2250 (style): Prefer putting braces around variable
references even when not strictly required.
^----------^ SC2086 (info): Double quote to
prevent globbing and word splitting.
^----------^ SC2250 (style): Prefer putting
braces around variable references even when not strictly required.
Did you mean:
echo -n "${RES}" | tee -a "${RESULT_FILE}"
In tools/emr_storage_regression/standard_set/run_queries.sh line 22:
[[ "$i" != "$TRIES" ]] && echo -n "," | tee -a $RESULT_FILE
^-- SC2250 (style): Prefer putting braces around variable
references even when not strictly required.
^----^ SC2250 (style): Prefer putting braces around
variable references even when not strictly required.
^----------^ SC2086
(info): Double quote to prevent globbing and word splitting.
^----------^ SC2250
(style): Prefer putting braces around variable references even when not
strictly required.
Did you mean:
[[ "${i}" != "${TRIES}" ]] && echo -n "," | tee -a "${RESULT_FILE}"
In tools/emr_storage_regression/standard_set/run_queries.sh line 24:
echo "" | tee -a $RESULT_FILE
^----------^ SC2086 (info): Double quote to prevent
globbing and word splitting.
^----------^ SC2250 (style): Prefer putting braces
around variable references even when not strictly required.
Did you mean:
echo "" | tee -a "${RESULT_FILE}"
In tools/emr_storage_regression/standard_set/run_queries.sh line 27:
done < $5
^-- SC2086 (info): Double quote to prevent globbing and word
splitting.
Did you mean:
done < "$5"
In tools/emr_storage_regression/standard_set/run_standard_set.sh line 7:
if [ -z "$4" ]; then
^---------^ SC2292 (style): Prefer [[ ]] over [ ] for tests in Bash/Ksh.
Did you mean:
if [[ -z "$4" ]]; then
In tools/emr_storage_regression/standard_set/run_standard_set.sh line 14:
if [ -z "$5" ]; then
^---------^ SC2292 (style): Prefer [[ ]] over [ ] for tests in Bash/Ksh.
Did you mean:
if [[ -z "$5" ]]; then
In tools/emr_storage_regression/standard_set/run_standard_set.sh line 25:
if [ "${case}" = 'ssb' ] ; then
^-------------------^ SC2292 (style): Prefer [[ ]] over [ ] for tests in
Bash/Ksh.
Did you mean:
if [[ "${case}" = 'ssb' ]] ; then
In tools/emr_storage_regression/standard_set/run_standard_set.sh line 29:
elif [ "${case}" = 'ssb_flat' ]; then
^------------------------^ SC2292 (style): Prefer [[ ]] over [ ] for
tests in Bash/Ksh.
Did you mean:
elif [[ "${case}" = 'ssb_flat' ]]; then
In tools/emr_storage_regression/standard_set/run_standard_set.sh line 33:
elif [ "${case}" = 'tpch' ]; then
^--------------------^ SC2292 (style): Prefer [[ ]] over [ ] for tests
in Bash/Ksh.
Did you mean:
elif [[ "${case}" = 'tpch' ]]; then
In tools/emr_storage_regression/standard_set/run_standard_set.sh line 37:
elif [ "${case}" = 'clickbench' ]; then
^--------------------------^ SC2292 (style): Prefer [[ ]] over [ ] for
tests in Bash/Ksh.
Did you mean:
elif [[ "${case}" = 'clickbench' ]]; then
For more information:
https://www.shellcheck.net/wiki/SC2148 -- Tips depend on target shell and
y...
https://www.shellcheck.net/wiki/SC2016 -- Expressions don't expand in
singl...
https://www.shellcheck.net/wiki/SC2086 -- Double quote to prevent globbing
...
----------
You can address the above issues in one of three ways:
1. Manually correct the issue in the offending shell script;
2. Disable specific issues by adding the comment:
# shellcheck disable=NNNN
above the line that contains the issue, where NNNN is the error code;
3. Add '-e NNNN' to the SHELLCHECK_OPTS setting in your .yml action file.
```
</details>
<details>
<summary>shfmt errors</summary>
```
'shfmt ' returned error 1 finding the following formatting issues:
----------
--- tools/emr_storage_regression/emr_tools.sh.orig
+++ tools/emr_storage_regression/emr_tools.sh
@@ -45,11 +45,11 @@
--profile)
PROFILE="$2"
if [[ -n "${PROFILE}" ]]; then
- echo "${PROFILE}"
- ENV="$3"
- if [[ -z "${ENV}" ]]; then
- echo "ENV is not specified, don't use profile"
- fi
+ echo "${PROFILE}"
+ ENV="$3"
+ if [[ -z "${ENV}" ]]; then
+ echo "ENV is not specified, don't use profile"
+ fi
fi
shift 2
;;
@@ -113,45 +113,45 @@
export FE_QUERY_PORT=${PORT}
if [[ ${CASE} == 'ping' ]]; then
- if [[ ${SERVICE} == 'hw' ]]; then
-
HMS_META_URI="thrift://node-master1rjzj.mrs-dr8h.com:9083,thrift://node-master2rksk.mrs-dr8h.com:9083"
-
BEELINE_URI="jdbc:hive2://192.168.0.8:2181/;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2;hive.server2.proxy.user=hive"
- HMS_WAREHOUSE=obs://datalake-bench/user
- elif [[ ${SERVICE} == 'ali' ]]; then
- HMS_META_URI="thrift://172.16.1.162:9083",
- HMS_WAREHOUSE=oss://benchmark-oss/user
- else
- # [[ ${SERVICE} == 'tx' ]];
- HMS_META_URI="thrift://172.21.0.32:7004"
- HMS_WAREHOUSE=cosn://datalake-bench-cos-1308700295/user
- fi
- sh ping_test/ping_poc.sh "${ENDPOINT}" "${REGION}" "${SERVICE}" "${AK}"
"${SK}" "${HMS_META_URI}" "${HMS_WAREHOUSE}" "${BEELINE_URI}"
+ if [[ ${SERVICE} == 'hw' ]]; then
+
HMS_META_URI="thrift://node-master1rjzj.mrs-dr8h.com:9083,thrift://node-master2rksk.mrs-dr8h.com:9083"
+
BEELINE_URI="jdbc:hive2://192.168.0.8:2181/;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2;hive.server2.proxy.user=hive"
+ HMS_WAREHOUSE=obs://datalake-bench/user
+ elif [[ ${SERVICE} == 'ali' ]]; then
+ HMS_META_URI="thrift://172.16.1.162:9083",
+ HMS_WAREHOUSE=oss://benchmark-oss/user
+ else
+ # [[ ${SERVICE} == 'tx' ]];
+ HMS_META_URI="thrift://172.21.0.32:7004"
+ HMS_WAREHOUSE=cosn://datalake-bench-cos-1308700295/user
+ fi
+ sh ping_test/ping_poc.sh "${ENDPOINT}" "${REGION}" "${SERVICE}" "${AK}"
"${SK}" "${HMS_META_URI}" "${HMS_WAREHOUSE}" "${BEELINE_URI}"
elif [[ ${CASE} == 'data_set' ]]; then
- if [[ ${SERVICE} == 'tx' ]]; then
- BUCKET=cosn://datalake-bench-cos-1308700295
- elif [[ ${SERVICE} == 'ali' ]]; then
- BUCKET=oss://benchmark-oss
- fi
- # gen table for spark
- if ! sh stardard_set/gen_spark_create_sql.sh "${BUCKET}" obj; then
- echo "Fail to generate spark obj table for test set"
- exit 1
- fi
- if ! sh stardard_set/gen_spark_create_sql.sh hdfs:///benchmark-hdfs hdfs;
then
- echo "Fail to generate spark hdfs table for test set, import hdfs data
first"
- exit 1
- fi
-
- # FE_HOST=172.16.1.163
- # USER=root
- # PORT=9035
- TYPE=hdfs sh stardard_set/run_standard_set.sh "${FE_HOST}" "${USER}"
"${PORT}" hms_hdfs
- TYPE=hdfs sh stardard_set/run_standard_set.sh "${FE_HOST}" "${USER}"
"${PORT}" iceberg_hms
- if [[ ${SERVICE} == 'tx' ]]; then
- sh stardard_set/run_standard_set.sh "${FE_HOST}" "${USER}" "${PORT}"
hms_cos
- sh stardard_set/run_standard_set.sh "${FE_HOST}" "${USER}" "${PORT}"
iceberg_hms_cos
- elif [[ ${SERVICE} == 'ali' ]]; then
- sh stardard_set/run_standard_set.sh "${FE_HOST}" "${USER}" "${PORT}"
hms_oss
- sh stardard_set/run_standard_set.sh "${FE_HOST}" "${USER}" "${PORT}"
iceberg_hms_oss
- fi
+ if [[ ${SERVICE} == 'tx' ]]; then
+ BUCKET=cosn://datalake-bench-cos-1308700295
+ elif [[ ${SERVICE} == 'ali' ]]; then
+ BUCKET=oss://benchmark-oss
+ fi
+ # gen table for spark
+ if ! sh stardard_set/gen_spark_create_sql.sh "${BUCKET}" obj; then
+ echo "Fail to generate spark obj table for test set"
+ exit 1
+ fi
+ if ! sh stardard_set/gen_spark_create_sql.sh hdfs:///benchmark-hdfs
hdfs; then
+ echo "Fail to generate spark hdfs table for test set, import hdfs
data first"
+ exit 1
+ fi
+
+ # FE_HOST=172.16.1.163
+ # USER=root
+ # PORT=9035
+ TYPE=hdfs sh stardard_set/run_standard_set.sh "${FE_HOST}" "${USER}"
"${PORT}" hms_hdfs
+ TYPE=hdfs sh stardard_set/run_standard_set.sh "${FE_HOST}" "${USER}"
"${PORT}" iceberg_hms
+ if [[ ${SERVICE} == 'tx' ]]; then
+ sh stardard_set/run_standard_set.sh "${FE_HOST}" "${USER}"
"${PORT}" hms_cos
+ sh stardard_set/run_standard_set.sh "${FE_HOST}" "${USER}"
"${PORT}" iceberg_hms_cos
+ elif [[ ${SERVICE} == 'ali' ]]; then
+ sh stardard_set/run_standard_set.sh "${FE_HOST}" "${USER}"
"${PORT}" hms_oss
+ sh stardard_set/run_standard_set.sh "${FE_HOST}" "${USER}"
"${PORT}" iceberg_hms_oss
+ fi
fi
--- tools/emr_storage_regression/ping_test/ping_poc.sh.orig
+++ tools/emr_storage_regression/ping_test/ping_poc.sh
@@ -14,8 +14,8 @@
USER=${USER}
if [[ -z ${HMS_WAREHOUSE} ]]; then
- echo "Need warehouse for ${SERVICE}"
-fi
+ echo "Need warehouse for ${SERVICE}"
+fi
cd "$(dirname "$0")" || exit
run_spark_create_sql() {
@@ -31,7 +31,7 @@
--conf spark.sql.catalog.hms.type=hive \
--conf spark.sql.defaultCatalog=hms \
--conf spark.sql.catalog.hms.warehouse=${HMS_WAREHOUSE} \
- -f data/create_spark_ping.sql" 2> spark_create.log
+ -f data/create_spark_ping.sql" 2>spark_create.log
elif [[ ${SERVICE} == 'tx' ]]; then
PARAM="--jars
/usr/local/service/iceberg/iceberg-spark-runtime-3.2_2.12-0.13.1.jar \
--conf
spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
\
@@ -40,9 +40,9 @@
--conf
spark.sql.catalog.local=org.apache.iceberg.spark.SparkCatalog \
--conf spark.sql.catalog.local.type=hadoop \
--conf spark.sql.catalog.local.warehouse=/usr/hive/warehouse \
- -f data/create_spark_ping.sql" 2> spark_create.log
+ -f data/create_spark_ping.sql" 2>spark_create.log
elif [[ ${SERVICE} == 'hw' ]]; then
- PARAM="-f data/create_spark_ping.sql" 2> spark_create.log
+ PARAM="-f data/create_spark_ping.sql" 2>spark_create.log
else
echo "Unknown service type: ${SERVICE}"
exit 1
@@ -52,18 +52,18 @@
run_spark_create_sql
run_hive_create_sql() {
- if [[ ${SERVICE} == 'hw' ]]; then
- beeline -u "${BEELINE_URI}" -f data/create_hive_ping.sql 2>
hive_create.log
- else
- hive -f data/create_hive_ping.sql 2> hive_create.log
- fi
+ if [[ ${SERVICE} == 'hw' ]]; then
+ beeline -u "${BEELINE_URI}" -f data/create_hive_ping.sql
2>hive_create.log
+ else
+ hive -f data/create_hive_ping.sql 2>hive_create.log
+ fi
}
run_hive_create_sql
## Step 2: make ping data
-spark-sql -f data_for_spark.sql > spark_data.log
-hive -f data_for_hive.sql > hive_data.log
+spark-sql -f data_for_spark.sql >spark_data.log
+hive -f data_for_hive.sql >hive_data.log
run_query() {
QUERY_NUM=1
@@ -74,46 +74,46 @@
echo -n "create catalog ${QUERY_NUM},"
for i in $(seq 1 "${TRIES}"); do
if [[ -n ${catalog} ]]; then
- query="switch ${catalog};${query}"
+ query="switch ${catalog};${query}"
fi
RES=$(mysql -vvv -h"${FE_HOST}" -u"${USER}"
-P"${FE_QUERY_PORT}" -e "${query}")
echo -n "${RES}"
[[ "${i}" != "${TRIES}" ]] && echo -n ","
done
QUERY_NUM=$((QUERY_NUM + 1))
- done < "${sql_file}"
+ done <"${sql_file}"
}
## Step 3: create external catalog in doris
case "${SERVICE}" in
- ali)
- sed -e 's#ENDPOINT#'"${ENDPOINT}"'#g' -e
's#META_URI#'"${HMS_META_URI}"'#g' -e 's#AK_INPUT#'"${AK}"'#g' -e
's#SK_INPUT#'"${SK}"'#g' create_catalog_aliyun.sql > emr_catalog.sql
- ## support dlf and jindo in beijing
- # DLF_ENDPOINT=datalake-vpc.cn-beijing.aliyuncs.com
- # sed -e 's#DLF_ENDPOINT#'"${DLF_ENDPOINT}"'#g'
create_catalog_aliyun.sql > emr_catalog.sql
- # JINDO_ENDPOINT=cn-beijing.oss-dls.aliyuncs.com
- # sed -e 's#JINDO_ENDPOINT#'"${JINDO_ENDPOINT}"'#g'
create_catalog_aliyun.sql > emr_catalog.sql
- ;;
- tx)
- sed -e 's#ENDPOINT#'"${ENDPOINT}"'#g' -e
's#META_URI#'"${HMS_META_URI}"'#g' -e 's#AK_INPUT#'"${AK}"'#g' -e
's#SK_INPUT#'"${SK}"'#g' create_catalog_tx.sql > emr_catalog.sql
- ;;
- aws)
- sed -e 's#ENDPOINT#'"${ENDPOINT}"'#g' -e
's#META_URI#'"${HMS_META_URI}"'#g' -e 's#AK_INPUT#'"${AK}"'#g' -e
's#SK_INPUT#'"${SK}"'#g' create_catalog_aws.sql > emr_catalog.sql
- ;;
- hw)
- sed -e 's#ENDPOINT#'"${ENDPOINT}"'#g' -e
's#META_URI#'"${HMS_META_URI}"'#g' -e 's#AK_INPUT#'"${AK}"'#g' -e
's#SK_INPUT#'"${SK}"'#g' create_catalog_hw.sql > emr_catalog.sql
- ;;
- *)
- echo "Internal error"
- exit 1
- ;;
+ali)
+ sed -e 's#ENDPOINT#'"${ENDPOINT}"'#g' -e
's#META_URI#'"${HMS_META_URI}"'#g' -e 's#AK_INPUT#'"${AK}"'#g' -e
's#SK_INPUT#'"${SK}"'#g' create_catalog_aliyun.sql >emr_catalog.sql
+ ## support dlf and jindo in beijing
+ # DLF_ENDPOINT=datalake-vpc.cn-beijing.aliyuncs.com
+ # sed -e 's#DLF_ENDPOINT#'"${DLF_ENDPOINT}"'#g'
create_catalog_aliyun.sql > emr_catalog.sql
+ # JINDO_ENDPOINT=cn-beijing.oss-dls.aliyuncs.com
+ # sed -e 's#JINDO_ENDPOINT#'"${JINDO_ENDPOINT}"'#g'
create_catalog_aliyun.sql > emr_catalog.sql
+ ;;
+tx)
+ sed -e 's#ENDPOINT#'"${ENDPOINT}"'#g' -e
's#META_URI#'"${HMS_META_URI}"'#g' -e 's#AK_INPUT#'"${AK}"'#g' -e
's#SK_INPUT#'"${SK}"'#g' create_catalog_tx.sql >emr_catalog.sql
+ ;;
+aws)
+ sed -e 's#ENDPOINT#'"${ENDPOINT}"'#g' -e
's#META_URI#'"${HMS_META_URI}"'#g' -e 's#AK_INPUT#'"${AK}"'#g' -e
's#SK_INPUT#'"${SK}"'#g' create_catalog_aws.sql >emr_catalog.sql
+ ;;
+hw)
+ sed -e 's#ENDPOINT#'"${ENDPOINT}"'#g' -e
's#META_URI#'"${HMS_META_URI}"'#g' -e 's#AK_INPUT#'"${AK}"'#g' -e
's#SK_INPUT#'"${SK}"'#g' create_catalog_hw.sql >emr_catalog.sql
+ ;;
+*)
+ echo "Internal error"
+ exit 1
+ ;;
esac
-
+
run_query emr_catalog.sql
## Step 4: query ping
EMR_CATALOG=$(awk '{print $6}' emr_catalog.sql)
# required echo here, or the EMR_CATALOG will not be split.
for c in $(echo "${EMR_CATALOG}"); do
- run_query ping.sql "${c}"
+ run_query ping.sql "${c}"
done
--- tools/emr_storage_regression/standard_set/gen_spark_create_sql.sh.orig
+++ tools/emr_storage_regression/standard_set/gen_spark_create_sql.sh
@@ -2,16 +2,15 @@
BUCKET=$1
TYPE=$2
cd "$(dirname "$0")" || exit
-sh gen_tbl/gen_ssb_create_sql.sh "${BUCKET}"/ssb/ssb100_orc
ssb100_orc_"${TYPE}" orc > create_"${TYPE}".sql
-sh gen_tbl/gen_ssb_create_sql.sh "${BUCKET}"/ssb/ssb100_parquet
ssb100_parquet_"${TYPE}" parquet >> create_"${TYPE}".sql
+sh gen_tbl/gen_ssb_create_sql.sh "${BUCKET}"/ssb/ssb100_orc
ssb100_orc_"${TYPE}" orc >create_"${TYPE}".sql
+sh gen_tbl/gen_ssb_create_sql.sh "${BUCKET}"/ssb/ssb100_parquet
ssb100_parquet_"${TYPE}" parquet >>create_"${TYPE}".sql
# tpch
-sh gen_tbl/gen_tpch_create_sql.sh "${BUCKET}"/tpch/tpch100_orc
tpch100_orc_"${TYPE}" orc >> create_"${TYPE}".sql
-sh gen_tbl/gen_tpch_create_sql.sh "${BUCKET}"/tpch/tpch100_parquet
tpch100_parquet_"${TYPE}" parquet >> create_"${TYPE}".sql
+sh gen_tbl/gen_tpch_create_sql.sh "${BUCKET}"/tpch/tpch100_orc
tpch100_orc_"${TYPE}" orc >>create_"${TYPE}".sql
+sh gen_tbl/gen_tpch_create_sql.sh "${BUCKET}"/tpch/tpch100_parquet
tpch100_parquet_"${TYPE}" parquet >>create_"${TYPE}".sql
# clickbench
-sh gen_tbl/gen_clickbench_create_sql.sh "${BUCKET}"/clickbench/hits_parquet
clickbench_parquet_"${TYPE}" parquet >> create_"${TYPE}".sql
-sh gen_tbl/gen_clickbench_create_sql.sh "${BUCKET}"/clickbench/hits_orc
clickbench_orc_"${TYPE}" orc >> create_"${TYPE}".sql
+sh gen_tbl/gen_clickbench_create_sql.sh "${BUCKET}"/clickbench/hits_parquet
clickbench_parquet_"${TYPE}" parquet >>create_"${TYPE}".sql
+sh gen_tbl/gen_clickbench_create_sql.sh "${BUCKET}"/clickbench/hits_orc
clickbench_orc_"${TYPE}" orc >>create_"${TYPE}".sql
# iceberg
# sh gen_tbl/gen_ssb_create_sql.sh oss://benchmark-oss/ssb/ssb100_iceberg
ssb100_iceberg iceberg >> create_"${TYPE}".sql
# sh gen_tbl/gen_tpch_create_sql.sh
oss://benchmark-oss/tpch/tpch100_iceberg tpch100_iceberg iceberg >>
create_"${TYPE}".sql
# sh gen_tbl/gen_clickbench_create_sql.sh
oss://benchmark-oss/clickbench/hits_iceberg clickbench_iceberg_hdfs >>
create_"${TYPE}".sql
-
---
tools/emr_storage_regression/standard_set/gen_tbl/gen_clickbench_create_sql.sh.orig
+++
tools/emr_storage_regression/standard_set/gen_tbl/gen_clickbench_create_sql.sh
@@ -2,7 +2,7 @@
if [ -z $1 ]; then
echo 'the first argument is database location'
exit
-else
+else
db_loc=$1
fi
---
tools/emr_storage_regression/standard_set/gen_tbl/gen_ssb_create_sql.sh.orig
+++ tools/emr_storage_regression/standard_set/gen_tbl/gen_ssb_create_sql.sh
@@ -2,7 +2,7 @@
if [ -z $1 ]; then
echo 'the first argument is database location'
exit
-else
+else
db_loc=$1
fi
---
tools/emr_storage_regression/standard_set/gen_tbl/gen_tpch_create_sql.sh.orig
+++ tools/emr_storage_regression/standard_set/gen_tbl/gen_tpch_create_sql.sh
@@ -2,7 +2,7 @@
if [ -z $1 ]; then
echo 'the first argument is database location'
exit
-else
+else
db_loc=$1
fi
--- tools/emr_storage_regression/standard_set/run_queries.sh.orig
+++ tools/emr_storage_regression/standard_set/run_queries.sh
@@ -17,11 +17,11 @@
while read -r query; do
echo -n "query${QUERY_NUM}," | tee -a $RESULT_FILE
for i in $(seq 1 $TRIES); do
- RES=$( mysql -vvv -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}"
-D"${DB}" -e "${query}" | perl -nle 'print $1 if /((\d+\.\d+)+ sec)/' || :)
+ RES=$(mysql -vvv -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}"
-D"${DB}" -e "${query}" | perl -nle 'print $1 if /((\d+\.\d+)+ sec)/' || :)
echo -n "$RES" | tee -a $RESULT_FILE
[[ "$i" != "$TRIES" ]] && echo -n "," | tee -a $RESULT_FILE
done
echo "" | tee -a $RESULT_FILE
QUERY_NUM=$((QUERY_NUM + 1))
-done < $5
+done <$5
--- tools/emr_storage_regression/standard_set/run_standard_set.sh.orig
+++ tools/emr_storage_regression/standard_set/run_standard_set.sh
@@ -7,22 +7,22 @@
if [ -z "$4" ]; then
echo 'need catalog name'
exit
-else
+else
catalog_name=$4
fi
if [ -z "$5" ]; then
echo "run all test"
-else
+else
case=$5
fi
if [[ -z ${TYPE} ]]; then
- TYPE=obj
+ TYPE=obj
fi
echo "execute ${case} benchmark for ${TYPE}..."
-if [ "${case}" = 'ssb' ] ; then
+if [ "${case}" = 'ssb' ]; then
# ssb
sh run_queries.sh "${FE_HOST}" "${USER}" "${PORT}"
"${catalog_name}".ssb100_parquet_"${TYPE}" queries/ssb_queries.sql
sh run_queries.sh "${FE_HOST}" "${USER}" "${PORT}"
"${catalog_name}".ssb100_orc_"${TYPE}" queries/ssb_queries.sql
----------
You can reformat the above files to meet shfmt's requirements by typing:
shfmt -w filename
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
