This is an automated email from the ASF dual-hosted git repository.
capistrant pushed a commit to branch 34.0.0
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/34.0.0 by this push:
new 2b478401fd5 Use openjdk:17-slim as base image for IT (#18251) (#18256)
2b478401fd5 is described below
commit 2b478401fd55638ed529af27e450422a699d419b
Author: Lucas Capistrant <[email protected]>
AuthorDate: Tue Jul 15 11:16:08 2025 -0500
Use openjdk:17-slim as base image for IT (#18251) (#18256)
Co-authored-by: Cece Mei <[email protected]>
---
integration-tests/docker/Dockerfile | 30 +++++++++-------------
integration-tests/docker/druid.sh | 27 +++++++++----------
integration-tests/docker/run-mysql.sh | 11 +++++++-
integration-tests/pom.xml | 2 ++
.../script/copy_resources_template.sh | 23 +++++++++++------
.../script/docker_build_containers.sh | 2 +-
6 files changed, 54 insertions(+), 41 deletions(-)
diff --git a/integration-tests/docker/Dockerfile
b/integration-tests/docker/Dockerfile
index 2e8a73c35f0..9c41a41baff 100644
--- a/integration-tests/docker/Dockerfile
+++ b/integration-tests/docker/Dockerfile
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-ARG JDK_VERSION=17-slim-buster
+ARG JDK_VERSION=17-slim-bullseye
FROM openjdk:$JDK_VERSION as druidbase
# Bundle everything into one script so cleanup can reduce image size.
@@ -27,7 +27,6 @@ ARG ZK_VERSION
ARG APACHE_ARCHIVE_MIRROR_HOST=https://downloads.apache.org
RUN APACHE_ARCHIVE_MIRROR_HOST=${APACHE_ARCHIVE_MIRROR_HOST}
/root/base-setup.sh && rm -f /root/base-setup.sh
-
FROM druidbase
ARG MYSQL_VERSION
ARG MARIA_VERSION
@@ -37,14 +36,6 @@ ARG CONFLUENT_VERSION
# Verify Java version
RUN java -version
-RUN echo "[mysqld]\ncharacter-set-server=utf8\ncollation-server=utf8_bin\n" >>
/etc/mysql/my.cnf
-
-# Setup metadata store
-# touch is needed because OverlayFS's copy-up operation breaks POSIX
standards. See https://github.com/docker/for-linux/issues/72.
-RUN find /var/lib/mysql -type f -exec touch {} \; && /etc/init.d/mysql start \
- && echo "CREATE USER 'druid'@'%' IDENTIFIED BY 'diurd'; GRANT ALL ON
druid.* TO 'druid'@'%'; CREATE database druid DEFAULT CHARACTER SET utf8mb4;" |
mysql -u root \
- && /etc/init.d/mysql stop
-
# Add Druid scripts and jars
ADD bin/* /usr/local/druid/bin/
ADD lib/* /usr/local/druid/lib/
@@ -65,11 +56,17 @@ RUN if [ "$MYSQL_DRIVER_CLASSNAME" =
"com.mysql.jdbc.Driver" ] ; \
RUN wget -q
"https://packages.confluent.io/maven/io/confluent/kafka-protobuf-provider/$CONFLUENT_VERSION/kafka-protobuf-provider-$CONFLUENT_VERSION.jar";
\
-O /usr/local/druid/lib/kafka-protobuf-provider.jar
-# Add sample data
-# touch is needed because OverlayFS's copy-up operation breaks POSIX
standards. See https://github.com/docker/for-linux/issues/72.
-RUN find /var/lib/mysql -type f -exec touch {} \; && service mysql start \
- && /usr/local/druid/bin/run-java -cp "/usr/local/druid/lib/*"
-Ddruid.extensions.directory=/usr/local/druid/extensions
-Ddruid.extensions.loadList='["mysql-metadata-storage"]'
-Ddruid.metadata.storage.type=mysql
-Ddruid.metadata.mysql.driver.driverClassName=$MYSQL_DRIVER_CLASSNAME
org.apache.druid.cli.Main tools metadata-init
--connectURI="jdbc:mysql://localhost:3306/druid" --user=druid --password=diurd \
- && /etc/init.d/mysql stop
+# Setup metadata store
+ADD run-mysql.sh /run-mysql.sh
+RUN echo "[mysqld]\ncharacter-set-server=utf8\ncollation-server=utf8_bin\n" >>
/etc/mysql/my.cnf
+RUN mkdir -p /run/mysqld && chown mysql:mysql /run/mysqld
+RUN ( \
+ bash /run-mysql.sh; \
+ echo "CREATE USER 'druid'@'%' IDENTIFIED BY 'diurd'; GRANT ALL ON druid.*
TO 'druid'@'%'; CREATE database druid DEFAULT CHARACTER SET utf8mb4;" | mysql
-u root; \
+ /usr/local/druid/bin/run-java -cp "/usr/local/druid/lib/*"
-Ddruid.extensions.directory=/usr/local/druid/extensions
-Ddruid.extensions.loadList='["mysql-metadata-storage"]'
-Ddruid.metadata.storage.type=mysql
-Ddruid.metadata.mysql.driver.driverClassName=$MYSQL_DRIVER_CLASSNAME
org.apache.druid.cli.Main tools metadata-init
--connectURI="jdbc:mysql://localhost:3306/druid" --user=druid --password=diurd;
\
+ mysqladmin shutdown \
+ )
+
ADD test-data /test-data
# Setup supervisord
@@ -78,9 +75,6 @@ ADD supervisord.conf /etc/supervisor/conf.d/supervisord.conf
# Add druid configuration setup script
ADD druid.sh /druid.sh
-# mysql
-ADD run-mysql.sh /run-mysql.sh
-
# internal docker_ip:9092 endpoint is used to access Kafka from other Docker
containers
# external docker ip:9093 endpoint is used to access Kafka from test code
# run this last to avoid rebuilding the image every time the ip changes
diff --git a/integration-tests/docker/druid.sh
b/integration-tests/docker/druid.sh
index d7bd80ca32e..973d7055ca3 100755
--- a/integration-tests/docker/druid.sh
+++ b/integration-tests/docker/druid.sh
@@ -78,30 +78,31 @@ setupConfig()
var=$(echo "$evar" | sed -e 's?^\([^=]*\)=.*?\1?g' -e 's?_?.?g')
setKey $DRUID_SERVICE "$var" "$val"
done
+ if [ "$MYSQL_DRIVER_CLASSNAME" != "com.mysql.jdbc.Driver" ] ; then
+ setKey $DRUID_SERVICE druid.metadata.mysql.driver.driverClassName
$MYSQL_DRIVER_CLASSNAME
+ fi
}
setupData()
{
+ # note: this function exists for legacy reasons, ideally we should do data
insert in IT's setup method.
+ if [ -n "$DRUID_SERVICE" ]; then
+ echo "DRUID_SERVICE is set, skipping data setup"
+ return
+ fi
+ # note: this function exists for legacy reasons, ideally we should do data
insert in IT's setup method.
+
+ bash /run-mysql.sh
# The "query" and "security" test groups require data to be setup before
running the tests.
# In particular, they requires segments to be download from a pre-existing
s3 bucket.
# This is done by using the loadSpec put into metadatastore and s3
credientials set below.
if [ "$DRUID_INTEGRATION_TEST_GROUP" = "query" ] || [
"$DRUID_INTEGRATION_TEST_GROUP" = "query-retry" ] || [
"$DRUID_INTEGRATION_TEST_GROUP" = "query-error" ] || [
"$DRUID_INTEGRATION_TEST_GROUP" = "high-availability" ] || [
"$DRUID_INTEGRATION_TEST_GROUP" = "security" ] || [
"$DRUID_INTEGRATION_TEST_GROUP" = "ldap-security" ] || [
"$DRUID_INTEGRATION_TEST_GROUP" = "upgrade" ] || [
"$DRUID_INTEGRATION_TEST_GROUP" = "centralized-datasource-schema" ] || [
"$DRUID_INTEGRATION_TEST_GROUP" [...]
- # touch is needed because OverlayFS's copy-up operation breaks POSIX
standards. See https://github.com/docker/for-linux/issues/72.
- find /var/lib/mysql -type f -exec touch {} \; && service mysql start \
- && cat /test-data/${DRUID_INTEGRATION_TEST_GROUP}-sample-data.sql |
mysql -u root druid \
- && /etc/init.d/mysql stop
- fi
-
- if [ "$MYSQL_DRIVER_CLASSNAME" != "com.mysql.jdbc.Driver" ] ; then
- setKey $DRUID_SERVICE druid.metadata.mysql.driver.driverClassName
$MYSQL_DRIVER_CLASSNAME
+ cat /test-data/${DRUID_INTEGRATION_TEST_GROUP}-sample-data.sql | mysql -u
root druid
fi
# The SqlInputSource tests in the "input-source" test group require data to
be setup in MySQL before running the tests.
if [ "$DRUID_INTEGRATION_TEST_GROUP" = "input-source" ] ; then
- # touch is needed because OverlayFS's copy-up operation breaks POSIX
standards. See https://github.com/docker/for-linux/issues/72.
- find /var/lib/mysql -type f -exec touch {} \; && service mysql start \
- && echo "GRANT ALL ON sqlinputsource.* TO 'druid'@'%'; CREATE database
sqlinputsource DEFAULT CHARACTER SET utf8mb4;" | mysql -u root druid \
- && cat /test-data/sql-input-source-sample-data.sql | mysql -u root
druid \
- && /etc/init.d/mysql stop
+ echo "GRANT ALL ON sqlinputsource.* TO 'druid'@'%'; CREATE database
sqlinputsource DEFAULT CHARACTER SET utf8mb4;" | mysql -u root druid
+ cat /test-data/sql-input-source-sample-data.sql | mysql -u root druid
fi
}
diff --git a/integration-tests/docker/run-mysql.sh
b/integration-tests/docker/run-mysql.sh
index 7faf12ed0f9..aa5d165a62e 100755
--- a/integration-tests/docker/run-mysql.sh
+++ b/integration-tests/docker/run-mysql.sh
@@ -15,4 +15,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-find /var/lib/mysql -type f -exec touch {} \; && /usr/bin/pidproxy
/var/run/mysqld/mysqld.pid /usr/bin/mysqld_safe --bind-address=0.0.0.0
\ No newline at end of file
+(mysqld --bind-address=0.0.0.0 || true) & # there might already be a mysqld
running, so we ignore the error
+/usr/bin/pidproxy /var/run/mysqld/mysqld.pid /usr/bin/mysqld_safe
--bind-address=0.0.0.0
+echo "Waiting for MySQL to be ready...";
+for i in {30..0}; do
+ mysqladmin ping --silent && break;
+ sleep 1; \
+done; \
+if [ "$i" = 0 ]; then
+ echo "MySQL did not start"; exit 1;
+fi;
diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml
index 5a6a4c99061..595a79c64b4 100644
--- a/integration-tests/pom.xml
+++ b/integration-tests/pom.xml
@@ -569,6 +569,7 @@
<start.hadoop.docker>false</start.hadoop.docker>
<docker.run.skip>false</docker.run.skip>
<docker.build.skip>false</docker.build.skip>
+
<docker.build.skipMavenBuild>false</docker.build.skipMavenBuild>
<docker.build.hadoop>false</docker.build.hadoop>
<it.indexer>middleManager</it.indexer>
<override.config.path />
@@ -598,6 +599,7 @@
<DRUID_INTEGRATION_TEST_OVERRIDE_CONFIG_PATH>${override.config.path}</DRUID_INTEGRATION_TEST_OVERRIDE_CONFIG_PATH>
<DRUID_INTEGRATION_TEST_RESOURCE_FILE_DIR_PATH>${resource.file.dir.path}</DRUID_INTEGRATION_TEST_RESOURCE_FILE_DIR_PATH>
<DRUID_INTEGRATION_TEST_SKIP_BUILD_DOCKER>${docker.build.skip}</DRUID_INTEGRATION_TEST_SKIP_BUILD_DOCKER>
+
<DRUID_INTEGRATION_TEST_SKIP_MAVEN_BUILD_DOCKER>${docker.build.skipMavenBuild}</DRUID_INTEGRATION_TEST_SKIP_MAVEN_BUILD_DOCKER>
<DRUID_INTEGRATION_TEST_SKIP_RUN_DOCKER>${docker.run.skip}</DRUID_INTEGRATION_TEST_SKIP_RUN_DOCKER>
<DRUID_INTEGRATION_TEST_INDEXER>${it.indexer}</DRUID_INTEGRATION_TEST_INDEXER>
<MYSQL_VERSION>${mysql.version}</MYSQL_VERSION>
diff --git a/integration-tests/script/copy_resources_template.sh
b/integration-tests/script/copy_resources_template.sh
index 7f1c4180247..f5fa127b0c4 100755
--- a/integration-tests/script/copy_resources_template.sh
+++ b/integration-tests/script/copy_resources_template.sh
@@ -28,14 +28,21 @@ rm -rf $SHARED_DIR/docker
mkdir -p $SHARED_DIR
cp -R docker $SHARED_DIR/docker
-pushd ../
-rm -rf distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin
-# using parallel build here may not yield significant speedups
-mvn -B -Pskip-static-checks,skip-tests -Dweb.console.skip=true install
-Pintegration-test
-mv distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin/bin
$SHARED_DIR/docker/bin
-mv distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin/lib
$SHARED_DIR/docker/lib
-mv
distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin/extensions
$SHARED_DIR/docker/extensions
-popd
+if [ -z "$DRUID_INTEGRATION_TEST_SKIP_MAVEN_BUILD_DOCKER" ] || [
"$DRUID_INTEGRATION_TEST_SKIP_MAVEN_BUILD_DOCKER" = "false" ]
+then
+ pushd ../
+ rm -rf distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin
+ # using parallel build here may not yield significant speedups
+ mvn -B -Pskip-static-checks,skip-tests -Dweb.console.skip=true install
-Pintegration-test
+ mv distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin/bin
$SHARED_DIR/docker/bin
+ mv distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin/lib
$SHARED_DIR/docker/lib
+ mv
distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin/extensions
$SHARED_DIR/docker/extensions
+ popd
+else
+ cp -R
../distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin/bin
$SHARED_DIR/docker/bin
+ cp -R
../distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin/lib
$SHARED_DIR/docker/lib
+ cp -R
../distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin/extensions
$SHARED_DIR/docker/extensions
+fi
# Make directoriess if they dont exist
mkdir -p $SHARED_DIR/hadoop_xml
diff --git a/integration-tests/script/docker_build_containers.sh
b/integration-tests/script/docker_build_containers.sh
index bed98c7db5f..9a5214ffaa6 100755
--- a/integration-tests/script/docker_build_containers.sh
+++ b/integration-tests/script/docker_build_containers.sh
@@ -29,7 +29,7 @@ else
11 | 17 | 21)
echo "Build druid-cluster with Java $DRUID_INTEGRATION_TEST_JVM_RUNTIME"
docker build -t druid/cluster \
- --build-arg JDK_VERSION=$DRUID_INTEGRATION_TEST_JVM_RUNTIME-slim-buster \
+ --build-arg
JDK_VERSION=$DRUID_INTEGRATION_TEST_JVM_RUNTIME-slim-bullseye \
--build-arg ZK_VERSION \
--build-arg KAFKA_VERSION \
--build-arg CONFLUENT_VERSION \
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
