This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit a8b66e5f1d63bda435c979d337a17d950aedb702
Author: Tim Armstrong <tarmstr...@cloudera.com>
AuthorDate: Thu Jan 17 12:10:03 2019 -0800
IMPALA-8066: Build coordinator and executor containers
The containers are essentially the same except for -is_executor
and -is_coordinator flags and the open ports (executors don't
need to expose HS2 and Beeswax). Over time we may want to
specialize the configurations further.
Building separate containers on top of impala_base is lightweight enough
and this a) reduces the amount of configuration required and b) makes it
clear which ports should open. It will also nudge people in the direction
of using dedicated coordinators and executors in Kubernetes, which I
believe is the right approach.
The previous impalad container was renamed to impalad_coord_exec
to be unambiguous.
Change-Id: I22f8ded167179478d7556f612b8b3e9d1b019a7a
Reviewed-on: http://gerrit.cloudera.org:8080/12228
Reviewed-by: Tim Armstrong <tarmstr...@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com>
---
docker/CMakeLists.txt | 54 +++++++++++++++++++++++-------
docker/catalogd/Dockerfile | 2 +-
docker/{impalad => coord_exec}/Dockerfile | 2 +-
docker/{impalad => coordinator}/Dockerfile | 4 +--
docker/{impalad => executor}/Dockerfile | 8 ++---
docker/statestored/Dockerfile | 2 +-
6 files changed, 49 insertions(+), 23 deletions(-)
diff --git a/docker/CMakeLists.txt b/docker/CMakeLists.txt
index ae34dd6..382407c 100644
--- a/docker/CMakeLists.txt
+++ b/docker/CMakeLists.txt
@@ -30,7 +30,8 @@ add_custom_command(
OUTPUT ${IMPALA_BASE_BUILD_CONTEXT_TIMESTAMP}
COMMAND ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py
COMMAND touch ${IMPALA_BASE_BUILD_CONTEXT_TIMESTAMP}
- DEPENDS daemons fe test-configuration
${CMAKE_SOURCE_DIR}/docker/setup_build_context.py
+ DEPENDS daemons fe ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py
+ DEPENDS ${CMAKE_SOURCE_DIR}/fe/src/test/resources/.test_config_timestamp
COMMENT "Creating impala base build context."
VERBATIM
)
@@ -54,20 +55,48 @@ add_custom_target(impala_base_image
DEPENDS ${IMPALA_BASE_IMAGE_TIMESTAMP}
)
-# Command and target for the Impalad image.
-set(IMPALAD_IMAGE_TIMESTAMP
- ${CMAKE_SOURCE_DIR}/docker/.impalad_image_timestamp
+# Command and target for the various Impalad images.
+set(COORD_EXEC_IMAGE_TIMESTAMP
+ ${CMAKE_SOURCE_DIR}/docker/.coord_exec_image_timestamp
)
add_custom_command(
- OUTPUT ${IMPALAD_IMAGE_TIMESTAMP}
- COMMAND cd ${CMAKE_SOURCE_DIR}/docker/impalad && docker build -t impalad .
- COMMAND touch ${IMPALAD_IMAGE_TIMESTAMP}
- DEPENDS ${IMPALA_BASE_IMAGE_TIMESTAMP}
${CMAKE_SOURCE_DIR}/docker/impalad/Dockerfile
- COMMENT "Building Impalad docker image."
+ OUTPUT ${COORD_EXEC_IMAGE_TIMESTAMP}
+ COMMAND cd ${CMAKE_SOURCE_DIR}/docker/coord_exec && docker build -t
impalad_coord_exec .
+ COMMAND touch ${COORD_EXEC_IMAGE_TIMESTAMP}
+ DEPENDS ${IMPALA_BASE_IMAGE_TIMESTAMP}
${CMAKE_SOURCE_DIR}/docker/coord_exec/Dockerfile
+ COMMENT "Building Impalad Coord Exec docker image."
VERBATIM
)
-add_custom_target(impalad_image
- DEPENDS ${IMPALAD_IMAGE_TIMESTAMP}
+add_custom_target(coord_exec_image
+ DEPENDS ${COORD_EXEC_IMAGE_TIMESTAMP}
+)
+set(COORDINATOR_IMAGE_TIMESTAMP
+ ${CMAKE_SOURCE_DIR}/docker/.coordinator_image_timestamp
+)
+add_custom_command(
+ OUTPUT ${COORDINATOR_IMAGE_TIMESTAMP}
+ COMMAND cd ${CMAKE_SOURCE_DIR}/docker/coordinator && docker build -t
impalad_coordinator .
+ COMMAND touch ${COORDINATOR_IMAGE_TIMESTAMP}
+ DEPENDS ${IMPALA_BASE_IMAGE_TIMESTAMP}
${CMAKE_SOURCE_DIR}/docker/coordinator/Dockerfile
+ COMMENT "Building Impalad Coordinator docker image."
+ VERBATIM
+)
+add_custom_target(coordinator_image
+ DEPENDS ${COORDINATOR_IMAGE_TIMESTAMP}
+)
+set(EXECUTOR_IMAGE_TIMESTAMP
+ ${CMAKE_SOURCE_DIR}/docker/.executor_image_timestamp
+)
+add_custom_command(
+ OUTPUT ${EXECUTOR_IMAGE_TIMESTAMP}
+ COMMAND cd ${CMAKE_SOURCE_DIR}/docker/executor && docker build -t
impalad_executor .
+ COMMAND touch ${EXECUTOR_IMAGE_TIMESTAMP}
+ DEPENDS ${IMPALA_BASE_IMAGE_TIMESTAMP}
${CMAKE_SOURCE_DIR}/docker/executor/Dockerfile
+ COMMENT "Building Impalad Executor docker image."
+ VERBATIM
+)
+add_custom_target(executor_image
+ DEPENDS ${EXECUTOR_IMAGE_TIMESTAMP}
)
# Command and target for the Statestored image.
@@ -102,5 +131,6 @@ add_custom_target(catalogd_image
DEPENDS ${CATALOGD_IMAGE_TIMESTAMP}
)
add_custom_target(docker_images
- DEPENDS impalad_image statestored_image catalogd_image
+ DEPENDS coord_exec_image coordinator_image executor_image
+ DEPENDS statestored_image catalogd_image
)
diff --git a/docker/catalogd/Dockerfile b/docker/catalogd/Dockerfile
index b7728cd..8699fb3 100644
--- a/docker/catalogd/Dockerfile
+++ b/docker/catalogd/Dockerfile
@@ -21,5 +21,5 @@ FROM impala_base
# Debug webserver
EXPOSE 25020
-CMD ["/opt/impala/bin/daemon_entrypoint.sh", "/opt/impala/bin/catalogd",\
+ENTRYPOINT ["/opt/impala/bin/daemon_entrypoint.sh",
"/opt/impala/bin/catalogd",\
"-abort_on_config_error=false", "-state_store_host=statestored"]
diff --git a/docker/impalad/Dockerfile b/docker/coord_exec/Dockerfile
similarity index 92%
copy from docker/impalad/Dockerfile
copy to docker/coord_exec/Dockerfile
index 8e2f615..8000b32 100644
--- a/docker/impalad/Dockerfile
+++ b/docker/coord_exec/Dockerfile
@@ -25,6 +25,6 @@ EXPOSE 21050
# Debug webserver
EXPOSE 25000
-CMD ["/opt/impala/bin/daemon_entrypoint.sh", "/opt/impala/bin/impalad",\
+ENTRYPOINT ["/opt/impala/bin/daemon_entrypoint.sh", "/opt/impala/bin/impalad",\
"-abort_on_config_error=false", "-state_store_host=statestored",\
"-catalog_service_host=catalogd"]
diff --git a/docker/impalad/Dockerfile b/docker/coordinator/Dockerfile
similarity index 87%
copy from docker/impalad/Dockerfile
copy to docker/coordinator/Dockerfile
index 8e2f615..24a4cb4 100644
--- a/docker/impalad/Dockerfile
+++ b/docker/coordinator/Dockerfile
@@ -25,6 +25,6 @@ EXPOSE 21050
# Debug webserver
EXPOSE 25000
-CMD ["/opt/impala/bin/daemon_entrypoint.sh", "/opt/impala/bin/impalad",\
+ENTRYPOINT ["/opt/impala/bin/daemon_entrypoint.sh", "/opt/impala/bin/impalad",\
"-abort_on_config_error=false", "-state_store_host=statestored",\
- "-catalog_service_host=catalogd"]
+ "-catalog_service_host=catalogd" "-is_executor=false"]
diff --git a/docker/impalad/Dockerfile b/docker/executor/Dockerfile
similarity index 85%
rename from docker/impalad/Dockerfile
rename to docker/executor/Dockerfile
index 8e2f615..dc47fe1 100644
--- a/docker/impalad/Dockerfile
+++ b/docker/executor/Dockerfile
@@ -18,13 +18,9 @@
FROM impala_base
# Externally-facing ports
-# Beeswax
-EXPOSE 21000
-# HS2
-EXPOSE 21050
# Debug webserver
EXPOSE 25000
-CMD ["/opt/impala/bin/daemon_entrypoint.sh", "/opt/impala/bin/impalad",\
+ENTRYPOINT ["/opt/impala/bin/daemon_entrypoint.sh", "/opt/impala/bin/impalad",\
"-abort_on_config_error=false", "-state_store_host=statestored",\
- "-catalog_service_host=catalogd"]
+ "-catalog_service_host=catalogd" "-is_coordinator=false"]
diff --git a/docker/statestored/Dockerfile b/docker/statestored/Dockerfile
index 1264814..b29cb22 100644
--- a/docker/statestored/Dockerfile
+++ b/docker/statestored/Dockerfile
@@ -21,4 +21,4 @@ FROM impala_base
# Debug webserver
EXPOSE 25010
-CMD ["/opt/impala/bin/daemon_entrypoint.sh", "/opt/impala/bin/statestored"]
+ENTRYPOINT ["/opt/impala/bin/daemon_entrypoint.sh",
"/opt/impala/bin/statestored"]
