This is an automated email from the ASF dual-hosted git repository. tarmstrong pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit a8b66e5f1d63bda435c979d337a17d950aedb702 Author: Tim Armstrong <tarmstr...@cloudera.com> AuthorDate: Thu Jan 17 12:10:03 2019 -0800 IMPALA-8066: Build coordinator and executor containers The containers are essentially the same except for -is_executor and -is_coordinator flags and the open ports (executors don't need to expose HS2 and Beeswax). Over time we may want to specialize the configurations further. Building separate containers on top of impala_base is lightweight enough and this a) reduces the amount of configuration required and b) makes it clear which ports should open. It will also nudge people in the direction of using dedicated coordinators and executors in Kubernetes, which I believe is the right approach. The previous impalad container was renamed to impalad_coord_exec to be unambiguous. Change-Id: I22f8ded167179478d7556f612b8b3e9d1b019a7a Reviewed-on: http://gerrit.cloudera.org:8080/12228 Reviewed-by: Tim Armstrong <tarmstr...@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com> --- docker/CMakeLists.txt | 54 +++++++++++++++++++++++------- docker/catalogd/Dockerfile | 2 +- docker/{impalad => coord_exec}/Dockerfile | 2 +- docker/{impalad => coordinator}/Dockerfile | 4 +-- docker/{impalad => executor}/Dockerfile | 8 ++--- docker/statestored/Dockerfile | 2 +- 6 files changed, 49 insertions(+), 23 deletions(-) diff --git a/docker/CMakeLists.txt b/docker/CMakeLists.txt index ae34dd6..382407c 100644 --- a/docker/CMakeLists.txt +++ b/docker/CMakeLists.txt @@ -30,7 +30,8 @@ add_custom_command( OUTPUT ${IMPALA_BASE_BUILD_CONTEXT_TIMESTAMP} COMMAND ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py COMMAND touch ${IMPALA_BASE_BUILD_CONTEXT_TIMESTAMP} - DEPENDS daemons fe test-configuration ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py + DEPENDS daemons fe ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py + DEPENDS ${CMAKE_SOURCE_DIR}/fe/src/test/resources/.test_config_timestamp COMMENT "Creating impala base build context." VERBATIM ) @@ -54,20 +55,48 @@ add_custom_target(impala_base_image DEPENDS ${IMPALA_BASE_IMAGE_TIMESTAMP} ) -# Command and target for the Impalad image. -set(IMPALAD_IMAGE_TIMESTAMP - ${CMAKE_SOURCE_DIR}/docker/.impalad_image_timestamp +# Command and target for the various Impalad images. +set(COORD_EXEC_IMAGE_TIMESTAMP + ${CMAKE_SOURCE_DIR}/docker/.coord_exec_image_timestamp ) add_custom_command( - OUTPUT ${IMPALAD_IMAGE_TIMESTAMP} - COMMAND cd ${CMAKE_SOURCE_DIR}/docker/impalad && docker build -t impalad . - COMMAND touch ${IMPALAD_IMAGE_TIMESTAMP} - DEPENDS ${IMPALA_BASE_IMAGE_TIMESTAMP} ${CMAKE_SOURCE_DIR}/docker/impalad/Dockerfile - COMMENT "Building Impalad docker image." + OUTPUT ${COORD_EXEC_IMAGE_TIMESTAMP} + COMMAND cd ${CMAKE_SOURCE_DIR}/docker/coord_exec && docker build -t impalad_coord_exec . + COMMAND touch ${COORD_EXEC_IMAGE_TIMESTAMP} + DEPENDS ${IMPALA_BASE_IMAGE_TIMESTAMP} ${CMAKE_SOURCE_DIR}/docker/coord_exec/Dockerfile + COMMENT "Building Impalad Coord Exec docker image." VERBATIM ) -add_custom_target(impalad_image - DEPENDS ${IMPALAD_IMAGE_TIMESTAMP} +add_custom_target(coord_exec_image + DEPENDS ${COORD_EXEC_IMAGE_TIMESTAMP} +) +set(COORDINATOR_IMAGE_TIMESTAMP + ${CMAKE_SOURCE_DIR}/docker/.coordinator_image_timestamp +) +add_custom_command( + OUTPUT ${COORDINATOR_IMAGE_TIMESTAMP} + COMMAND cd ${CMAKE_SOURCE_DIR}/docker/coordinator && docker build -t impalad_coordinator . + COMMAND touch ${COORDINATOR_IMAGE_TIMESTAMP} + DEPENDS ${IMPALA_BASE_IMAGE_TIMESTAMP} ${CMAKE_SOURCE_DIR}/docker/coordinator/Dockerfile + COMMENT "Building Impalad Coordinator docker image." + VERBATIM +) +add_custom_target(coordinator_image + DEPENDS ${COORDINATOR_IMAGE_TIMESTAMP} +) +set(EXECUTOR_IMAGE_TIMESTAMP + ${CMAKE_SOURCE_DIR}/docker/.executor_image_timestamp +) +add_custom_command( + OUTPUT ${EXECUTOR_IMAGE_TIMESTAMP} + COMMAND cd ${CMAKE_SOURCE_DIR}/docker/executor && docker build -t impalad_executor . + COMMAND touch ${EXECUTOR_IMAGE_TIMESTAMP} + DEPENDS ${IMPALA_BASE_IMAGE_TIMESTAMP} ${CMAKE_SOURCE_DIR}/docker/executor/Dockerfile + COMMENT "Building Impalad Executor docker image." + VERBATIM +) +add_custom_target(executor_image + DEPENDS ${EXECUTOR_IMAGE_TIMESTAMP} ) # Command and target for the Statestored image. @@ -102,5 +131,6 @@ add_custom_target(catalogd_image DEPENDS ${CATALOGD_IMAGE_TIMESTAMP} ) add_custom_target(docker_images - DEPENDS impalad_image statestored_image catalogd_image + DEPENDS coord_exec_image coordinator_image executor_image + DEPENDS statestored_image catalogd_image ) diff --git a/docker/catalogd/Dockerfile b/docker/catalogd/Dockerfile index b7728cd..8699fb3 100644 --- a/docker/catalogd/Dockerfile +++ b/docker/catalogd/Dockerfile @@ -21,5 +21,5 @@ FROM impala_base # Debug webserver EXPOSE 25020 -CMD ["/opt/impala/bin/daemon_entrypoint.sh", "/opt/impala/bin/catalogd",\ +ENTRYPOINT ["/opt/impala/bin/daemon_entrypoint.sh", "/opt/impala/bin/catalogd",\ "-abort_on_config_error=false", "-state_store_host=statestored"] diff --git a/docker/impalad/Dockerfile b/docker/coord_exec/Dockerfile similarity index 92% copy from docker/impalad/Dockerfile copy to docker/coord_exec/Dockerfile index 8e2f615..8000b32 100644 --- a/docker/impalad/Dockerfile +++ b/docker/coord_exec/Dockerfile @@ -25,6 +25,6 @@ EXPOSE 21050 # Debug webserver EXPOSE 25000 -CMD ["/opt/impala/bin/daemon_entrypoint.sh", "/opt/impala/bin/impalad",\ +ENTRYPOINT ["/opt/impala/bin/daemon_entrypoint.sh", "/opt/impala/bin/impalad",\ "-abort_on_config_error=false", "-state_store_host=statestored",\ "-catalog_service_host=catalogd"] diff --git a/docker/impalad/Dockerfile b/docker/coordinator/Dockerfile similarity index 87% copy from docker/impalad/Dockerfile copy to docker/coordinator/Dockerfile index 8e2f615..24a4cb4 100644 --- a/docker/impalad/Dockerfile +++ b/docker/coordinator/Dockerfile @@ -25,6 +25,6 @@ EXPOSE 21050 # Debug webserver EXPOSE 25000 -CMD ["/opt/impala/bin/daemon_entrypoint.sh", "/opt/impala/bin/impalad",\ +ENTRYPOINT ["/opt/impala/bin/daemon_entrypoint.sh", "/opt/impala/bin/impalad",\ "-abort_on_config_error=false", "-state_store_host=statestored",\ - "-catalog_service_host=catalogd"] + "-catalog_service_host=catalogd" "-is_executor=false"] diff --git a/docker/impalad/Dockerfile b/docker/executor/Dockerfile similarity index 85% rename from docker/impalad/Dockerfile rename to docker/executor/Dockerfile index 8e2f615..dc47fe1 100644 --- a/docker/impalad/Dockerfile +++ b/docker/executor/Dockerfile @@ -18,13 +18,9 @@ FROM impala_base # Externally-facing ports -# Beeswax -EXPOSE 21000 -# HS2 -EXPOSE 21050 # Debug webserver EXPOSE 25000 -CMD ["/opt/impala/bin/daemon_entrypoint.sh", "/opt/impala/bin/impalad",\ +ENTRYPOINT ["/opt/impala/bin/daemon_entrypoint.sh", "/opt/impala/bin/impalad",\ "-abort_on_config_error=false", "-state_store_host=statestored",\ - "-catalog_service_host=catalogd"] + "-catalog_service_host=catalogd" "-is_coordinator=false"] diff --git a/docker/statestored/Dockerfile b/docker/statestored/Dockerfile index 1264814..b29cb22 100644 --- a/docker/statestored/Dockerfile +++ b/docker/statestored/Dockerfile @@ -21,4 +21,4 @@ FROM impala_base # Debug webserver EXPOSE 25010 -CMD ["/opt/impala/bin/daemon_entrypoint.sh", "/opt/impala/bin/statestored"] +ENTRYPOINT ["/opt/impala/bin/daemon_entrypoint.sh", "/opt/impala/bin/statestored"]