This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 15acf46cc21 [opt](hive docker)Exit on creating table failed (#47390)
15acf46cc21 is described below
commit 15acf46cc2135f01cb30d7ab7b2cf419991dd44b
Author: Thearas <[email protected]>
AuthorDate: Sun Jan 26 15:10:18 2025 +0900
[opt](hive docker)Exit on creating table failed (#47390)
### What problem does this PR solve?
Problem Summary:
There're some table not found error when querying external hive catalog
in CI, like
[#614159](http://43.132.222.7:8111/buildConfiguration/Doris_External_Regression/614159?buildTab=overview&hideTestsFromDependencies=false&hideProblemsFromDependencies=false&expandPull+Request+Details=true&expandBuildTestsSection=true&expandBuildChangesSection=true&expandBuildDeploymentsSection=false)
and
[#613945](http://43.132.222.7:8111/buildConfiguration/Doris_External_Regression/613945?buildTab=tests&status=failed&expandedTest=build%3A%28id%3A613945%29%2Cid%3A2000000035).
<img width="782" alt="image"
src="https://github.com/user-attachments/assets/74ce8b6f-3116-4aca-a5a5-3d7f3845a1c9";
/>
The create table script should exit once the sql failed, so we can
easily to see which script failed.
#### Failed scripts
Total 7 ~ 10+ failures, you can search `FAILED: ` in [hive metastore
docker
logs](https://github.com/user-attachments/files/18532083/logs.txt).
1. Create table when hive metastore is not ready , like `student`:
```bash
2025-01-24T04:30:01.794131928Z FAILED: SemanticException
org.apache.hadoop.hive.ql.metadata.HiveException:
java.lang.RuntimeException: Unable to instantiate
org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient
```
2. HQL grammar wrong, like
`/mnt/scripts/data/regression/crdmm_data/run.sh` :
```bash
FAILED: ParseException line 3:12 extraneous input 'crdmm_data' expecting
EOF near '<EOF>'
```
3. Error at `msck repair table partition_location_2;`, need `set
hive.msck.path.validation=ignore;`, like
`/mnt/scripts/data/multi_catalog/partition_location_2/run.sh`:
```bash
FAILED: Execution Error, return code 1 from
org.apache.hadoop.hive.ql.exec.DDLTask
```
---
.../docker-compose/hive/hive-2x.yaml.tpl | 3 +++
.../docker-compose/hive/hive-3x.yaml.tpl | 3 +++
.../partition_location_1/create_table.hql | 3 ++-
.../partition_location_2/create_table.hql | 1 +
.../data/regression/crdmm_data/create_table.hql | 2 +-
.../docker-compose/hive/scripts/hive-metastore.sh | 28 +++++++++++-----------
6 files changed, 24 insertions(+), 16 deletions(-)
diff --git a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
index 10285d7e405..88a9597032b 100644
--- a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
@@ -21,6 +21,7 @@ version: "3.8"
services:
namenode:
image: bde2020/hadoop-namenode:2.0.0-hadoop2.7.4-java8
+ restart: always
environment:
- CLUSTER_NAME=test
env_file:
@@ -38,6 +39,7 @@ services:
datanode:
image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8
+ restart: always
env_file:
- ./hadoop-hive-2x.env
environment:
@@ -82,6 +84,7 @@ services:
command: /bin/bash /mnt/scripts/hive-metastore.sh
environment:
SERVICE_PRECONDITION: "${IP_HOST}:50070 ${IP_HOST}:50075
${IP_HOST}:${PG_PORT}"
+ HMS_PORT: "${HMS_PORT}"
container_name: ${CONTAINER_UID}hive2-metastore
expose:
- "${HMS_PORT}"
diff --git a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
index 7fdac05f71c..84d4a653554 100644
--- a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
@@ -21,6 +21,7 @@ version: "3.8"
services:
namenode:
image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
+ restart: always
environment:
- CLUSTER_NAME=test
env_file:
@@ -38,6 +39,7 @@ services:
datanode:
image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
+ restart: always
env_file:
- ./hadoop-hive-3x.env
environment:
@@ -82,6 +84,7 @@ services:
command: /bin/bash /mnt/scripts/hive-metastore.sh
environment:
SERVICE_PRECONDITION: "${IP_HOST}:9870 ${IP_HOST}:9864
${IP_HOST}:${PG_PORT}"
+ HMS_PORT: "${HMS_PORT}"
container_name: ${CONTAINER_UID}hive3-metastore
expose:
- "${HMS_PORT}"
diff --git
a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/partition_location_1/create_table.hql
b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/partition_location_1/create_table.hql
index a477daf78ea..79ba42ac23f 100644
---
a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/partition_location_1/create_table.hql
+++
b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/partition_location_1/create_table.hql
@@ -17,6 +17,7 @@ TBLPROPERTIES (
'transient_lastDdlTime'='1682405696');
ALTER TABLE partition_location_1 ADD PARTITION (part='part1') LOCATION
'/user/doris/suites/multi_catalog/partition_location_1/part=part1';
-ALTER TABLE partition_location_1 ADD PARTITION (part='part2') LOCATION
'/user/doris/suites/multi_catalog/partition_location_1/20230425';
+ALTER TABLE partition_location_1 ADD PARTITION (part='part2') LOCATION
'/user/doris/suites/multi_catalog/partition_location_1/part=part2';
+set hive.msck.path.validation=ignore;
msck repair table partition_location_1;
diff --git
a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/partition_location_2/create_table.hql
b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/partition_location_2/create_table.hql
index bc40eb85f1c..9c3c025954a 100644
---
a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/partition_location_2/create_table.hql
+++
b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/partition_location_2/create_table.hql
@@ -20,4 +20,5 @@ TBLPROPERTIES (
ALTER TABLE partition_location_2 ADD PARTITION (part1='part1_1',
part2='part2_1') LOCATION
'/user/doris/suites/multi_catalog/partition_location_2/part1=part1_1/part2=part2_1';
ALTER TABLE partition_location_2 ADD PARTITION (part1='part1_2',
part2='part2_2') LOCATION
'/user/doris/suites/multi_catalog/partition_location_2/20230425';
+set hive.msck.path.validation=ignore;
msck repair table partition_location_2;
diff --git
a/docker/thirdparties/docker-compose/hive/scripts/data/regression/crdmm_data/create_table.hql
b/docker/thirdparties/docker-compose/hive/scripts/data/regression/crdmm_data/create_table.hql
index 0abba278308..e38d7c2cad1 100644
---
a/docker/thirdparties/docker-compose/hive/scripts/data/regression/crdmm_data/create_table.hql
+++
b/docker/thirdparties/docker-compose/hive/scripts/data/regression/crdmm_data/create_table.hql
@@ -158,5 +158,5 @@ LOCATION
TBLPROPERTIES (
'transient_lastDdlTime'='1685331029');
-msck repair crdmm_data;
+msck repair table crdmm_data;
diff --git a/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh
b/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh
index 842af6b78b2..ac4c9ae4480 100755
--- a/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh
+++ b/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh
@@ -22,26 +22,26 @@ parallel=$(getconf _NPROCESSORS_ONLN)
nohup /opt/hive/bin/hive --service metastore &
-# wait metastore start
-sleep 10s
-
-# create tables for other cases
-# new cases should use separate dir
-hadoop fs -mkdir -p /user/doris/suites/
-
-lockfile1="/mnt/scripts/run-data.lock"
-
# wait lockfile
+lockfile1="/mnt/scripts/run-data.lock"
while [[ -f "${lockfile1}" ]]; do
sleep 10
done
-
touch "${lockfile1}"
+# wait metastore start
+while ! $(nc -z localhost "${HMS_PORT:-9083}"); do
+ sleep 5s
+done
+
+# create tables for other cases
+# new cases should use separate dir
+hadoop fs -mkdir -p /user/doris/suites/
+
DATA_DIR="/mnt/scripts/data/"
-find "${DATA_DIR}" -type f -name "run.sh" -print0 | xargs -0 -n 1 -P
"${parallel}" -I {} sh -c '
+find "${DATA_DIR}" -type f -name "run.sh" -print0 | xargs -0 -n 1 -P
"${parallel}" -I {} bash -ec '
START_TIME=$(date +%s)
- chmod +x "{}" && "{}"
+ bash -e "{}" || (echo "Failed to executing script: {}" && exit 1)
END_TIME=$(date +%s)
EXECUTION_TIME=$((END_TIME - START_TIME))
echo "Script: {} executed in $EXECUTION_TIME seconds"
@@ -145,9 +145,9 @@ if [[ -z "$(hadoop fs -ls /user/doris/tvf_data)" ]]; then
fi
# create tables
-ls /mnt/scripts/create_preinstalled_scripts/*.hql | xargs -n 1 -P
"${parallel}" -I {} bash -c '
+ls /mnt/scripts/create_preinstalled_scripts/*.hql | xargs -n 1 -P
"${parallel}" -I {} bash -ec '
START_TIME=$(date +%s)
- hive -f {}
+ hive -f {} || (echo "Failed to executing hql: {}" && exit 1)
END_TIME=$(date +%s)
EXECUTION_TIME=$((END_TIME - START_TIME))
echo "Script: {} executed in $EXECUTION_TIME seconds"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
