This is an automated email from the ASF dual-hosted git repository. voonhous pushed a commit to branch release-1.2.1 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 71b90a97e900e758a4a6623f1bedfab23d55d2ba Author: voonhous <[email protected]> AuthorDate: Wed Jul 8 23:49:56 2026 +0800 test(integ-test): add Spark 4.1.1 stack to hive-sync E2E matrix (#19216) Adds a spark4.1 row to the integration-tests-hive-sync job, backed by new docker-compose_hadoop340_hive2310_spark411_{amd64,arm64}.yml files derived from the 4.0.2 pair (image tags and HDFS cluster name only; arm64 comment normalized to ASCII). VARIANT tests run on this row via the existing spark4 compose-prefix token; no Java harness changes. Requires apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-spark{base,master,worker,adhoc}_4.1.1 images to be published to Docker Hub before the new CI row can pass. (cherry picked from commit 1046ec084732e76075ed8b3b677a23a48503b4ae) --- .github/workflows/bot.yml | 12 +- ...r-compose_hadoop340_hive2310_spark411_amd64.yml | 267 +++++++++++++++++++++ ...r-compose_hadoop340_hive2310_spark411_arm64.yml | 267 +++++++++++++++++++++ 3 files changed, 543 insertions(+), 3 deletions(-) diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml index 851b53e78618..e4d3238ffc3f 100644 --- a/.github/workflows/bot.yml +++ b/.github/workflows/bot.yml @@ -1253,9 +1253,9 @@ jobs: integration-tests-hive-sync: # Testcontainers-based E2E hive sync coverage for Hudi's custom logical types - # (VECTOR, BLOB) and the Spark 4.0 VARIANT type. Runs the integ2 testcontainers - # suite (ITTestCustomTypeHiveSync) against a real Hive metastore on both a - # Spark 3.5.3 and Spark 4.0.2 stack. + # (VECTOR, BLOB) and the Spark 4.0+ VARIANT type. Runs the integ2 testcontainers + # suite (ITTestCustomTypeHiveSync) against a real Hive metastore on Spark 3.5.3, + # 4.0.2, and 4.1.1 stacks. runs-on: ubuntu-latest needs: changes strategy: @@ -1274,6 +1274,12 @@ jobs: jdkVersion: '17' composePrefix: 'docker-compose_hadoop340_hive2310_spark402' sparkAdhocImage: 'apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-sparkadhoc_4.0.2:latest' + - sparkProfile: 'spark4.1' + scalaProfile: '-Dscala-2.13 -Dscala.binary.version=2.13' + flinkProfile: 'flink1.20' + jdkVersion: '17' + composePrefix: 'docker-compose_hadoop340_hive2310_spark411' + sparkAdhocImage: 'apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-sparkadhoc_4.1.1:latest' steps: - if: needs.changes.outputs.relevant == 'true' uses: actions/checkout@v5 diff --git a/docker/compose/docker-compose_hadoop340_hive2310_spark411_amd64.yml b/docker/compose/docker-compose_hadoop340_hive2310_spark411_amd64.yml new file mode 100644 index 000000000000..15227b126c6b --- /dev/null +++ b/docker/compose/docker-compose_hadoop340_hive2310_spark411_amd64.yml @@ -0,0 +1,267 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + + namenode: + image: apachehudi/hudi-hadoop_3.4.0-namenode:latest + hostname: namenode + container_name: namenode + environment: + - CLUSTER_NAME=hudi_hadoop340_hive2310_spark411 + ports: + - "8020:8020" # HDFS NameNode IPC + - "9000:9000" # HDFS NameNode Client + - "9870:9870" # HDFS NameNode Web UI + env_file: + - ./hadoop.env + healthcheck: + test: ["CMD", "curl", "-f", "http://namenode:9870";] + interval: 30s + timeout: 10s + retries: 3 + + datanode1: + image: apachehudi/hudi-hadoop_3.4.0-datanode:latest + container_name: datanode1 + hostname: datanode1 + environment: + - CLUSTER_NAME=hudi_hadoop340_hive2310_spark411 + env_file: + - ./hadoop.env + ports: + - "50075:50075" + - "9864:9864" + - "50010:50010" + links: + - "namenode" + - "historyserver" + healthcheck: + test: ["CMD", "curl", "-f", "http://datanode1:9864";] + interval: 30s + timeout: 10s + retries: 3 + depends_on: + - namenode + + historyserver: + image: apachehudi/hudi-hadoop_3.4.0-history:latest + hostname: historyserver + container_name: historyserver + environment: + - CLUSTER_NAME=hudi_hadoop340_hive2310_spark411 + depends_on: + - "namenode" + links: + - "namenode" + ports: + - "8188:8188" + healthcheck: + test: ["CMD", "curl", "-f", "http://historyserver:8188";] + interval: 30s + timeout: 10s + retries: 3 + env_file: + - ./hadoop.env + volumes: + - historyserver:/hadoop/yarn/timeline + + # Pure Hive 2.3.10 stack (postgres 2.3 schema -> HMS 2.3.10 -> HS2 2.3.10). + # Matches hudi-spark-bundle's compile-time Hive 2.3 client, so Hudi hive-sync + # talks to HMS natively (no Thrift get_table incompat, no sharedPrefixes hack). + # Hadoop 3.4.0 HDFS is backward-compat for the 2.8.4-based Hive client. + hive-metastore-postgresql: + image: bde2020/hive-metastore-postgresql:2.3.0 + volumes: + - hive-metastore-postgresql:/var/lib/postgresql + hostname: hive-metastore-postgresql + container_name: hive-metastore-postgresql + + hivemetastore: + image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.10:latest + hostname: hivemetastore + container_name: hivemetastore + links: + - "hive-metastore-postgresql" + - "namenode" + env_file: + - ./hadoop.env + command: /opt/hive/bin/hive --service metastore + environment: + - "SERVICE_PRECONDITION=namenode:9870 hive-metastore-postgresql:5432" + ports: + - "9083:9083" + healthcheck: + test: ["CMD", "nc", "-z", "hivemetastore", "9083"] + interval: 30s + timeout: 10s + retries: 3 + depends_on: + - "hive-metastore-postgresql" + - "namenode" + + hiveserver: + image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.10:latest + hostname: hiveserver + container_name: hiveserver + env_file: + - ./hadoop.env + environment: + - SERVICE_PRECONDITION=hivemetastore:9083 + ports: + - "10000:10000" + depends_on: + - "hivemetastore" + links: + - "hivemetastore" + - "hive-metastore-postgresql" + - "namenode" + volumes: + - ${HUDI_WS}:/var/hoodie/ws + + zookeeper: + image: 'bitnamilegacy/zookeeper:3.6.4' + hostname: zookeeper + container_name: zookeeper + ports: + - "2181:2181" + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + + kafka: + image: 'bitnamilegacy/kafka:3.4.1' + hostname: kafkabroker + container_name: kafkabroker + ports: + - "9092:9092" + environment: + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - ALLOW_PLAINTEXT_LISTENER=yes + + sparkmaster: + image: apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-sparkmaster_4.1.1:latest + hostname: sparkmaster + container_name: sparkmaster + env_file: + - ./hadoop.env + ports: + - "8080:8080" + - "7077:7077" + - "8888:8888" + volumes: + - ${HUDI_WS}:/var/hoodie/ws + - ./notebooks:/opt/workspace/notebooks + environment: + - INIT_DAEMON_STEP=setup_spark + links: + - "hivemetastore" + - "hiveserver" + - "hive-metastore-postgresql" + - "namenode" + + spark-worker-1: + image: apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-sparkworker_4.1.1:latest + hostname: spark-worker-1 + container_name: spark-worker-1 + env_file: + - ./hadoop.env + depends_on: + - sparkmaster + ports: + - "8081:8081" + environment: + - SPARK_MASTER=spark://sparkmaster:7077 + links: + - "hivemetastore" + - "hiveserver" + - "hive-metastore-postgresql" + - "namenode" + + adhoc-1: + image: apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-sparkadhoc_4.1.1:latest + hostname: adhoc-1 + container_name: adhoc-1 + env_file: + - ./hadoop.env + depends_on: + - sparkmaster + ports: + - '4040:4040' + environment: + - SPARK_MASTER=spark://sparkmaster:7077 + links: + - "hivemetastore" + - "hiveserver" + - "hive-metastore-postgresql" + - "namenode" + volumes: + - ${HUDI_WS}:/var/hoodie/ws + + adhoc-2: + image: apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-sparkadhoc_4.1.1:latest + hostname: adhoc-2 + container_name: adhoc-2 + env_file: + - ./hadoop.env + depends_on: + - sparkmaster + environment: + - SPARK_MASTER=spark://sparkmaster:7077 + links: + - "hivemetastore" + - "hiveserver" + - "hive-metastore-postgresql" + - "namenode" + volumes: + - ${HUDI_WS}:/var/hoodie/ws + + minio: + image: 'minio/minio:latest' + hostname: minio + container_name: minio + ports: + - 9090:9090 # server address + - 9091:9091 # console address + volumes: + - minio-data:/data + environment: + - MINIO_ACCESS_KEY=minio + - MINIO_SECRET_KEY=minio123 + - MINIO_DOMAIN=minio + command: server --address ":9090" --console-address ":9091" /data + + mc: + image: minio/mc + container_name: mc + entrypoint: > + /bin/sh -c " + until (/usr/bin/mc alias set minio http://minio:9090 minio minio123 --api S3v4) do echo '...waiting...' && sleep 1; done; + /usr/bin/mc rm -r --force minio/warehouse; + /usr/bin/mc mb minio/warehouse; + /usr/bin/mc policy set public minio/warehouse; + tail -f /dev/null + " + depends_on: + - minio + +volumes: + namenode: + historyserver: + hive-metastore-postgresql: + minio-data: + +networks: + default: + name: hudi diff --git a/docker/compose/docker-compose_hadoop340_hive2310_spark411_arm64.yml b/docker/compose/docker-compose_hadoop340_hive2310_spark411_arm64.yml new file mode 100644 index 000000000000..15227b126c6b --- /dev/null +++ b/docker/compose/docker-compose_hadoop340_hive2310_spark411_arm64.yml @@ -0,0 +1,267 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + + namenode: + image: apachehudi/hudi-hadoop_3.4.0-namenode:latest + hostname: namenode + container_name: namenode + environment: + - CLUSTER_NAME=hudi_hadoop340_hive2310_spark411 + ports: + - "8020:8020" # HDFS NameNode IPC + - "9000:9000" # HDFS NameNode Client + - "9870:9870" # HDFS NameNode Web UI + env_file: + - ./hadoop.env + healthcheck: + test: ["CMD", "curl", "-f", "http://namenode:9870";] + interval: 30s + timeout: 10s + retries: 3 + + datanode1: + image: apachehudi/hudi-hadoop_3.4.0-datanode:latest + container_name: datanode1 + hostname: datanode1 + environment: + - CLUSTER_NAME=hudi_hadoop340_hive2310_spark411 + env_file: + - ./hadoop.env + ports: + - "50075:50075" + - "9864:9864" + - "50010:50010" + links: + - "namenode" + - "historyserver" + healthcheck: + test: ["CMD", "curl", "-f", "http://datanode1:9864";] + interval: 30s + timeout: 10s + retries: 3 + depends_on: + - namenode + + historyserver: + image: apachehudi/hudi-hadoop_3.4.0-history:latest + hostname: historyserver + container_name: historyserver + environment: + - CLUSTER_NAME=hudi_hadoop340_hive2310_spark411 + depends_on: + - "namenode" + links: + - "namenode" + ports: + - "8188:8188" + healthcheck: + test: ["CMD", "curl", "-f", "http://historyserver:8188";] + interval: 30s + timeout: 10s + retries: 3 + env_file: + - ./hadoop.env + volumes: + - historyserver:/hadoop/yarn/timeline + + # Pure Hive 2.3.10 stack (postgres 2.3 schema -> HMS 2.3.10 -> HS2 2.3.10). + # Matches hudi-spark-bundle's compile-time Hive 2.3 client, so Hudi hive-sync + # talks to HMS natively (no Thrift get_table incompat, no sharedPrefixes hack). + # Hadoop 3.4.0 HDFS is backward-compat for the 2.8.4-based Hive client. + hive-metastore-postgresql: + image: bde2020/hive-metastore-postgresql:2.3.0 + volumes: + - hive-metastore-postgresql:/var/lib/postgresql + hostname: hive-metastore-postgresql + container_name: hive-metastore-postgresql + + hivemetastore: + image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.10:latest + hostname: hivemetastore + container_name: hivemetastore + links: + - "hive-metastore-postgresql" + - "namenode" + env_file: + - ./hadoop.env + command: /opt/hive/bin/hive --service metastore + environment: + - "SERVICE_PRECONDITION=namenode:9870 hive-metastore-postgresql:5432" + ports: + - "9083:9083" + healthcheck: + test: ["CMD", "nc", "-z", "hivemetastore", "9083"] + interval: 30s + timeout: 10s + retries: 3 + depends_on: + - "hive-metastore-postgresql" + - "namenode" + + hiveserver: + image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.10:latest + hostname: hiveserver + container_name: hiveserver + env_file: + - ./hadoop.env + environment: + - SERVICE_PRECONDITION=hivemetastore:9083 + ports: + - "10000:10000" + depends_on: + - "hivemetastore" + links: + - "hivemetastore" + - "hive-metastore-postgresql" + - "namenode" + volumes: + - ${HUDI_WS}:/var/hoodie/ws + + zookeeper: + image: 'bitnamilegacy/zookeeper:3.6.4' + hostname: zookeeper + container_name: zookeeper + ports: + - "2181:2181" + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + + kafka: + image: 'bitnamilegacy/kafka:3.4.1' + hostname: kafkabroker + container_name: kafkabroker + ports: + - "9092:9092" + environment: + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - ALLOW_PLAINTEXT_LISTENER=yes + + sparkmaster: + image: apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-sparkmaster_4.1.1:latest + hostname: sparkmaster + container_name: sparkmaster + env_file: + - ./hadoop.env + ports: + - "8080:8080" + - "7077:7077" + - "8888:8888" + volumes: + - ${HUDI_WS}:/var/hoodie/ws + - ./notebooks:/opt/workspace/notebooks + environment: + - INIT_DAEMON_STEP=setup_spark + links: + - "hivemetastore" + - "hiveserver" + - "hive-metastore-postgresql" + - "namenode" + + spark-worker-1: + image: apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-sparkworker_4.1.1:latest + hostname: spark-worker-1 + container_name: spark-worker-1 + env_file: + - ./hadoop.env + depends_on: + - sparkmaster + ports: + - "8081:8081" + environment: + - SPARK_MASTER=spark://sparkmaster:7077 + links: + - "hivemetastore" + - "hiveserver" + - "hive-metastore-postgresql" + - "namenode" + + adhoc-1: + image: apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-sparkadhoc_4.1.1:latest + hostname: adhoc-1 + container_name: adhoc-1 + env_file: + - ./hadoop.env + depends_on: + - sparkmaster + ports: + - '4040:4040' + environment: + - SPARK_MASTER=spark://sparkmaster:7077 + links: + - "hivemetastore" + - "hiveserver" + - "hive-metastore-postgresql" + - "namenode" + volumes: + - ${HUDI_WS}:/var/hoodie/ws + + adhoc-2: + image: apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-sparkadhoc_4.1.1:latest + hostname: adhoc-2 + container_name: adhoc-2 + env_file: + - ./hadoop.env + depends_on: + - sparkmaster + environment: + - SPARK_MASTER=spark://sparkmaster:7077 + links: + - "hivemetastore" + - "hiveserver" + - "hive-metastore-postgresql" + - "namenode" + volumes: + - ${HUDI_WS}:/var/hoodie/ws + + minio: + image: 'minio/minio:latest' + hostname: minio + container_name: minio + ports: + - 9090:9090 # server address + - 9091:9091 # console address + volumes: + - minio-data:/data + environment: + - MINIO_ACCESS_KEY=minio + - MINIO_SECRET_KEY=minio123 + - MINIO_DOMAIN=minio + command: server --address ":9090" --console-address ":9091" /data + + mc: + image: minio/mc + container_name: mc + entrypoint: > + /bin/sh -c " + until (/usr/bin/mc alias set minio http://minio:9090 minio minio123 --api S3v4) do echo '...waiting...' && sleep 1; done; + /usr/bin/mc rm -r --force minio/warehouse; + /usr/bin/mc mb minio/warehouse; + /usr/bin/mc policy set public minio/warehouse; + tail -f /dev/null + " + depends_on: + - minio + +volumes: + namenode: + historyserver: + hive-metastore-postgresql: + minio-data: + +networks: + default: + name: hudi
