This is an automated email from the ASF dual-hosted git repository.
philo-he pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 0e55e98ede [GLUTEN-12850][VL] Support multiple Dev Container
configurations (#12853)
0e55e98ede is described below
commit 0e55e98ede0f9a9fa3b340432d439f37052eb510
Author: Felipe Pessoto <[email protected]>
AuthorDate: Thu Aug 27 17:20:34 2026 -0700
[GLUTEN-12850][VL] Support multiple Dev Container configurations (#12853)
---
.devcontainer/devcontainer.json | 3 +-
.devcontainer/post-create.sh | 148 ++++++++++++-
.devcontainer/{ => velox-static}/devcontainer.json | 37 ++--
docs/developers/NewToGluten.md | 6 +-
docs/developers/dev-container.md | 229 ++++++++++++---------
5 files changed, 289 insertions(+), 134 deletions(-)
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 81b75291cd..a709ed556d 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -1,4 +1,4 @@
-// Dev Container for the Gluten Velox backend.
+// Default Dev Container for daily Gluten Velox development.
// See docs/developers/dev-container.md for the rationale behind these settings
// and for the build commands to run once the container is up.
{
@@ -19,6 +19,7 @@
"containerEnv": {
"CCACHE_DIR": "/root/.ccache",
"CCACHE_MAXSIZE": "10G",
+ "GLUTEN_DEV_CONTAINER_VARIANT": "velox-dynamic",
// folly, gflags and glog are shared libraries in this image; a Velox
// dependency build that disagrees fails to link.
"VELOX_BUILD_SHARED": "ON"
diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh
index 6ca6215872..c1ae9f9af6 100755
--- a/.devcontainer/post-create.sh
+++ b/.devcontainer/post-create.sh
@@ -26,13 +26,65 @@
set -uo pipefail
NUM_THREADS_MARKER='# >>> gluten dev container num_threads >>>'
+STATIC_ARM_MARKER='# >>> gluten static dev container arm64 >>>'
+DEV_CONTAINER_VARIANT=${GLUTEN_DEV_CONTAINER_VARIANT:-velox-dynamic}
-warn() { echo "WARNING: $*" >&2; }
+WARNINGS=()
+warn() {
+ echo "WARNING: $*" >&2
+ WARNINGS+=("$*")
+}
echo "Preparing the Gluten dev container..."
-# Spark 4.0/4.1 and the UDF tests need JDK 17, which this JDK 8 image lacks.
-# Both JDKs can coexist: JAVA_HOME still points at JDK 8 for the default build.
+# The container runs as root while the bind-mounted workspace retains the host
+# user's ownership. Register only this repository so Git accepts that mismatch.
+WORKSPACE_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)
+if ! git config --global --get-all safe.directory 2>/dev/null |
+ grep -Fqx "$WORKSPACE_DIR"; then
+ git config --global --add safe.directory "$WORKSPACE_DIR" ||
+ warn "could not mark $WORKSPACE_DIR as a safe Git directory."
+fi
+
+# /workspaces persists across "Reopen in Container"/"Rebuild Container" and
+# across switching between the velox-dynamic and velox-static configs, but
+# ep/build-velox/build/velox_ep/_build and cpp/build bake in the vcpkg
+# toolchain choice (or its absence) at first CMake configure and CMake never
+# re-evaluates it. Reusing a build tree from the other variant does not error
+# clearly -- it silently resolves dependencies like zlib/zstd from the wrong
+# place and fails much later, e.g. "could not find SnappyConfig.cmake", deep
+# into a build that can take hours. Catch the mismatch up front instead.
+check_stale_build_tree() {
+ local cache="$1"
+ [ -f "$cache" ] || return 0
+ local has_toolchain=false
+ grep -q '^CMAKE_TOOLCHAIN_FILE:' "$cache" 2>/dev/null && has_toolchain=true
+
+ if [ "$DEV_CONTAINER_VARIANT" = "velox-static" ] && [ "$has_toolchain" =
false ]; then
+ warn "$cache was configured without the vcpkg toolchain (looks like it
came from the velox-dynamic container, or a build before --enable_vcpkg=ON).
Remove stale build trees before building here: rm -rf
ep/build-velox/build/velox_ep/_build cpp/build"
+ elif [ "$DEV_CONTAINER_VARIANT" = "velox-dynamic" ] && [ "$has_toolchain"
= true ]; then
+ warn "$cache was configured with the vcpkg toolchain (looks like it
came from the velox-static container). Remove stale build trees before building
here: rm -rf ep/build-velox/build/velox_ep/_build cpp/build"
+ fi
+}
+
+for cache in ep/build-velox/build/velox_ep/_build/*/CMakeCache.txt
cpp/build/CMakeCache.txt; do
+ check_stale_build_tree "$cache"
+done
+
+# vcpkg otherwise defaults to the x64 triplet on arm64.
+if [ "$DEV_CONTAINER_VARIANT" = "velox-static" ] &&
+ [ "$(uname -m)" = "aarch64" ] &&
+ ! grep -qF "$STATIC_ARM_MARKER" "$HOME/.bashrc" 2>/dev/null; then
+ cat >>"$HOME/.bashrc" <<EOF
+
+$STATIC_ARM_MARKER
+export CPU_TARGET=aarch64
+export VCPKG_FORCE_SYSTEM_BINARIES=1
+# <<< gluten static dev container arm64 <<<
+EOF
+fi
+
+# The dynamic image defaults to JDK 8; the static image already has JDK 17.
if [ ! -d /usr/lib/jvm/java-17-openjdk ]; then
echo "Installing JDK 17 alongside JDK 8 (needed for Spark 4.x)..."
dnf install -y --setopt=install_weak_deps=False java-17-openjdk-devel
>/dev/null ||
@@ -108,10 +160,22 @@ NUM_THREADS=$(/usr/local/bin/gluten-num-threads)
CPU_THREADS=$(nproc --ignore=2)
MEM_GB=$(awk '/^MemTotal:/ {printf "%d", $2 / 1048576}' /proc/meminfo
2>/dev/null)
-cat <<EOF
+print_parallelism() {
+ cat <<EOF
+NUM_THREADS=${NUM_THREADS} is exported for you, sized from this machine's
${MEM_GB:-?} GB at
+~4 GB per compile job. The build scripts would take $CPU_THREADS from the core
count
+alone, which invites the OOM killer. VS Code tasks do not read ~/.bashrc, so
pass
+--num_threads=${NUM_THREADS} there.
+
+EOF
+}
+
+case "$DEV_CONTAINER_VARIANT" in
+velox-dynamic)
+ cat <<EOF
============================================================================
-Gluten dev container is ready. The native build has NOT been run.
+Gluten Velox dynamic-link container is ready. The native build has NOT been
run.
Build the Velox backend and install the Gluten jars:
@@ -122,23 +186,31 @@ Build the Velox backend and install the Gluten jars:
--build_arrow=OFF Arrow is already installed under /usr/local
--build_tests=ON also build the C++ unit tests (drop it to build
faster)
--spark_version=3.5 build one Spark version instead of all five
+EOF
-NUM_THREADS=${NUM_THREADS} is exported for you, sized from this machine's
${MEM_GB:-?} GB at
-~4 GB per compile job. The build scripts would take $CPU_THREADS from the core
count
-alone, which invites the OOM killer. VS Code tasks do not read ~/.bashrc, so
pass
---num_threads=${NUM_THREADS} there.
+ print_parallelism
+ cat <<EOF
After changing C++ code, rebuild just the native side (drop build_velox when
only
Gluten's own C++ under cpp/ changed):
./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF \\
--build_tests=ON build_velox build_gluten_cpp
+Build for Spark 4.1 with JDK 17 (the Maven profile does not switch the running
JDK):
+
+ export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
+ export PATH="\$JAVA_HOME/bin:\$PATH"
+ java -version # must report 17
+ ./dev/buildbundle-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF \\
+ --build_tests=ON --spark_version=4.1
+
Run a Spark unit test suite (Spark distributions are pre-installed in this
image;
CI runs these on JDK 17, and without -DwildcardSuites the whole suite runs for
hours):
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
+ export PATH="\$JAVA_HOME/bin:\$PATH"
./build/mvn test -Pspark-ut -Pbackends-velox -Pspark-3.5 -Pjava-17 \\
-DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \\
-DwildcardSuites=org.apache.spark.sql.GlutenSQLQuerySuite
@@ -146,3 +218,61 @@ hours):
See docs/developers/dev-container.md for details.
============================================================================
EOF
+ ;;
+velox-static)
+ cat <<EOF
+============================================================================
+Gluten Velox static-link container is ready. The native build has NOT been run.
+
+Build a portable static-link jar:
+
+ ./dev/buildbundle-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF \\
+ --spark_version=3.5
+
+ --enable_vcpkg=ON statically link third-party dependencies
+ --build_arrow=OFF Arrow is already installed by the image
+ --spark_version=3.5 build one Spark version instead of all five
+
+S3, GCS, HDFS and ABFS are disabled by default. Enable only what you need with
+--enable_s3=ON, --enable_gcs=ON, --enable_hdfs=ON or --enable_abfs=ON.
+EOF
+
+ print_parallelism
+
+ cat <<EOF
+After changing C++ code, rebuild just the native side:
+
+ ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF \\
+ build_velox build_gluten_cpp
+
+JDK 17 is already active. To build for Spark 4.1:
+
+ java -version # must report 17
+ ./dev/buildbundle-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF \\
+ --spark_version=4.1
+
+This image does not include /opt/shims, so it is intended for static packaging
+and reproduction rather than Spark unit tests.
+
+See docs/developers/dev-container.md for details.
+============================================================================
+EOF
+ ;;
+*)
+ echo "ERROR: unknown GLUTEN_DEV_CONTAINER_VARIANT: $DEV_CONTAINER_VARIANT"
>&2
+ exit 1
+ ;;
+esac
+
+# Warnings logged with warn() above can easily scroll past unnoticed among all
+# the setup output, so repeat them here, in yellow, after everything else.
+if [ "${#WARNINGS[@]}" -gt 0 ]; then
+ YELLOW='\033[1;33m'
+ NO_COLOR='\033[0m'
+ echo -e
"${YELLOW}============================================================================${NO_COLOR}"
>&2
+ echo -e "${YELLOW}WARNINGS:${NO_COLOR}" >&2
+ for w in "${WARNINGS[@]}"; do
+ echo -e "${YELLOW} - $w${NO_COLOR}" >&2
+ done
+ echo -e
"${YELLOW}============================================================================${NO_COLOR}"
>&2
+fi
diff --git a/.devcontainer/devcontainer.json
b/.devcontainer/velox-static/devcontainer.json
similarity index 50%
copy from .devcontainer/devcontainer.json
copy to .devcontainer/velox-static/devcontainer.json
index 81b75291cd..e35bc6047c 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/velox-static/devcontainer.json
@@ -1,46 +1,36 @@
-// Dev Container for the Gluten Velox backend.
-// See docs/developers/dev-container.md for the rationale behind these settings
-// and for the build commands to run once the container is up.
+// Static-link environment for packaging and vcpkg issue reproduction.
+// See docs/developers/dev-container.md for build instructions and limitations.
{
- "name": "Gluten Velox Backend (centos-9, dynamic link)",
+ "name": "Gluten Velox Backend (centos-9, static link)",
- // Ships Velox's dependencies and Arrow under /usr/local, a pre-warmed
Maven
- // repository, and the Spark distributions gluten-ut needs under
/opt/shims.
- "image": "apache/gluten:centos-9-jdk8",
+ "image": "apache/gluten:vcpkg-centos-9",
- // Codespaces only. Storage is the binding constraint: a 32 GB disk cannot
- // hold the image plus the Velox build tree.
+ // Static linking needs substantially more memory than daily development.
"hostRequirements": {
- "cpus": 4,
- "memory": "16gb",
+ "cpus": 8,
+ "memory": "64gb",
"storage": "64gb"
},
"containerEnv": {
"CCACHE_DIR": "/root/.ccache",
"CCACHE_MAXSIZE": "10G",
- // folly, gflags and glog are shared libraries in this image; a Velox
- // dependency build that disagrees fails to link.
- "VELOX_BUILD_SHARED": "ON"
+ "GLUTEN_DEV_CONTAINER_VARIANT": "velox-static"
},
- // What /opt/rh/gcc-toolset-12/enable exports. The image's default GCC 11
- // cannot compile Velox's C++20 sources, and the build scripts do not
enable
- // the toolset themselves. Keep in sync with that script.
+ // Keep the compiler aligned with the toolchain used to build the vcpkg
cache.
"remoteEnv": {
"PATH": "/opt/rh/gcc-toolset-12/root/usr/bin:${containerEnv:PATH}",
"LD_LIBRARY_PATH":
"/opt/rh/gcc-toolset-12/root/usr/lib64:/opt/rh/gcc-toolset-12/root/usr/lib",
"PKG_CONFIG_PATH": "/opt/rh/gcc-toolset-12/root/usr/lib64/pkgconfig"
},
- // Survive "Rebuild Container". Docker seeds a new named volume from the
- // image, so the Maven repository it ships is kept, not hidden.
"mounts": [
- "source=gluten-ccache,target=/root/.ccache,type=volume",
- "source=gluten-m2,target=/root/.m2,type=volume"
+ "source=gluten-vcpkg-ccache,target=/root/.ccache,type=volume",
+ "source=gluten-vcpkg-m2,target=/root/.m2,type=volume",
+ "source=gluten-vcpkg-binary-cache,target=/var/cache/vcpkg,type=volume"
],
- // Prepares the environment only; see post-create.sh for why it never
builds.
"postCreateCommand": "bash .devcontainer/post-create.sh",
"customizations": {
@@ -51,8 +41,6 @@
"ms-python.python"
],
"settings": {
- // A build leaves >10 GB in the workspace; watching it exhausts
- // inotify handles.
"files.watcherExclude": {
"**/target/**": true,
"**/cpp/build/**": true,
@@ -69,7 +57,6 @@
"**/dev/vcpkg/.vcpkg/**": true,
"**/dev/vcpkg/vcpkg_installed/**": true
},
- // Written by the native build
(-DCMAKE_EXPORT_COMPILE_COMMANDS=ON).
"C_Cpp.default.compileCommands":
"${workspaceFolder}/cpp/build/compile_commands.json"
}
}
diff --git a/docs/developers/NewToGluten.md b/docs/developers/NewToGluten.md
index 27620395cf..16f0463247 100644
--- a/docs/developers/NewToGluten.md
+++ b/docs/developers/NewToGluten.md
@@ -44,9 +44,9 @@ Gluten requires GCC 11 or above.
### Dev Container
To skip the manual environment setup, you can develop Gluten inside a
pre-built Docker
-image using the [Dev Container](https://containers.dev/) configuration shipped
at
-`.devcontainer/devcontainer.json`. See [Dev Container](./dev-container.md) for
a brief
-guide and the list of available Gluten Docker images.
+image using the [Dev Container](https://containers.dev/) configurations under
+`.devcontainer/`. See [Dev Containers](./dev-container.md) for configuration,
+build and test instructions.
## Development
diff --git a/docs/developers/dev-container.md b/docs/developers/dev-container.md
index d4346533d9..bc8c0974d2 100644
--- a/docs/developers/dev-container.md
+++ b/docs/developers/dev-container.md
@@ -1,16 +1,15 @@
---
layout: page
-title: Dev Container
+title: Dev Containers
nav_order: 18
parent: Developer Overview
---
# Develop Gluten in a Dev Container
-Gluten ships a [Dev Container](https://containers.dev/) configuration at
-[`.devcontainer/devcontainer.json`](https://github.com/apache/gluten/blob/main/.devcontainer/devcontainer.json),
-so you can develop inside a pre-built image that already has the JDK, Maven,
GCC
-toolset and the whole Velox native dependency stack installed.
+Gluten provides Dev Container configurations for daily Velox development and
+static-link packaging. Both use pre-built images with the native dependency
+stack installed, and neither builds Gluten during container creation.
## Prerequisites
@@ -19,25 +18,34 @@ toolset and the whole Velox native dependency stack
installed.
[Dev Containers
extension](https://code.visualstudio.com/docs/devcontainers/containers),
or a [Codespaces](https://docs.github.com/en/codespaces)-enabled account.
-## Open the workspace
+## Choose a configuration
-In VS Code, run **Dev Containers: Reopen in Container** from the Command
Palette
-(`F1`). In Codespaces, create a Codespace for the repository.
+| Configuration | Path | Use case | Spark unit tests |
+|---|---|---|---|
+| **Velox dynamic link** (default) | `.devcontainer/devcontainer.json` | Daily
development | Yes |
+| **Velox static link** | `.devcontainer/velox-static/devcontainer.json` |
Portable jars and vcpkg issue reproduction | No (`/opt/shims` is not installed)
|
-## What the configuration does
+In VS Code, run **Dev Containers: Reopen in Container** from the Command
+Palette (`F1`) and select a configuration. In Codespaces, open **Create
+codespace with options** and select the configuration before creating the
+codespace.
-It opens the workspace in `apache/gluten:centos-9-jdk8` (CentOS Stream 9, JDK
8,
-**dynamically linked** dependencies), puts GCC 12 on `PATH` through `remoteEnv`
-because the default GCC 11 cannot compile Velox's C++20 sources, and runs
-[`.devcontainer/post-create.sh`](https://github.com/apache/gluten/blob/main/.devcontainer/post-create.sh),
-which installs JDK 17, `clang-format` 15 and the `regex` module, sizes
`NUM_THREADS`
-for the machine and prints the build commands.
+The shared
+[post-create
script](https://github.com/apache/gluten/blob/main/.devcontainer/post-create.sh)
+installs missing development tools, sizes `NUM_THREADS` for the machine and
+prints commands for the selected configuration.
-**The native build is not run automatically.** It takes tens of minutes to
several
-hours, which would stall container creation and leave a half-built tree behind
-whenever the editor disconnects or a Codespace times out.
+**The native build is not run automatically.** It takes tens of minutes to
+several hours, which would stall container creation and leave a half-built tree
+behind if the editor disconnects or a Codespace times out.
-## Build Gluten
+## Velox dynamic-link development
+
+The default configuration uses `apache/gluten:centos-9-jdk8`. It includes
+dynamically linked dependencies, Arrow under `/usr/local`, a pre-warmed Maven
+repository and Spark distributions under `/opt/shims`.
+
+Build the native backend and the Spark 3.5 jars:
```bash
./dev/buildbundle-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF \
@@ -46,12 +54,37 @@ whenever the editor disconnects or a Codespace times out.
| Flag | Why |
|---|---|
-| `--run_setup_script=OFF` | Velox's third-party libraries are already
installed in the image; `ON` rebuilds them all from source into `/usr/local`. |
+| `--run_setup_script=OFF` | Velox's third-party libraries are already
installed; `ON` rebuilds them from source. |
| `--build_arrow=OFF` | Arrow is already installed under `/usr/local` and its
jars are in `~/.m2`. |
| `--build_tests=ON` | Also builds the C++ unit tests. Drop it if you only
need the jars. |
-| `--spark_version=3.5` | The default, `ALL`, runs five full Maven builds
(Spark 3.3 to 4.1). |
+| `--spark_version=3.5` | The default, `ALL`, runs five Maven builds (Spark
3.3 to 4.1). |
+
+### Build for Spark 4.1
-To rebuild only the native side after a C++ change:
+Spark 4.0/4.1 require JDK 17 and Scala 2.13. The dynamic image defaults to JDK
+8, while `post-create.sh` installs JDK 17 alongside it. Switch the running JDK
+before starting the build:
+
+```bash
+export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
+export PATH="$JAVA_HOME/bin:$PATH"
+java -version # must report 17
+
+./dev/buildbundle-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF \
+ --build_tests=ON --spark_version=4.1
+```
+
+`buildbundle-veloxbe.sh` adds `-Pjava-17`, `-Pscala-2.13` and the Java 17
+release target for Spark 4.x, but Maven profiles cannot switch the JDK that is
+already running. If JDK 8 remains active, Scala fails with:
+
+```text
+scalac error: '17' is not a valid choice for '-release'
+```
+
+### Rebuild native code
+
+After changing C++ code:
```bash
./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF \
@@ -59,108 +92,112 @@ To rebuild only the native side after a C++ change:
```
Drop `build_velox` when only Gluten's own C++ under `cpp/` changed. Keep
-`--build_tests` matched with the flag you built with: `build_gluten_cpp` wipes
-`cpp/build` on every run, so omitting it silently drops the C++ test binaries.
+`--build_tests` matched with the original build: `build_gluten_cpp` wipes
+`cpp/build`, so omitting it also removes the C++ test binaries.
-Spark 4.0/4.1 need JDK 17 and Scala 2.13. `buildbundle-veloxbe.sh` adds the
Maven
-profiles, but `JAVA_HOME` is yours to set:
+### Run tests
+
+Run a Spark 3.5 suite on JDK 17:
```bash
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
-./dev/buildbundle-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF
--spark_version=4.0
+export PATH="$JAVA_HOME/bin:$PATH"
+
+./build/mvn test -Pspark-ut -Pbackends-velox -Pspark-3.5 -Pjava-17 \
+ -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \
+ -DwildcardSuites=org.apache.spark.sql.GlutenSQLQuerySuite
```
-### Build parallelism and the OOM killer
+`-DwildcardSuites` takes a fully qualified class name and keeps a run to
+minutes. Do not add `-pl gluten-ut`: it selects only the aggregator POM and no
+test suite runs. See [HowTo](./HowTo.md#3-how-to-debug-javascala) for more.
-`builddeps-veloxbe.sh` sizes `NUM_THREADS` as `nproc --ignore=2`, ignoring
memory,
-while Velox's heavier translation units peak at ~3.5 GB resident each. On a
32-core,
-62 GB container that is 30 jobs asking for roughly 100 GB, and the OOM killer
takes
-down the build or the container. `post-create.sh` therefore exports a value
allowing
-~4 GB per job — 13 jobs on that machine, measured at a 41 GB peak.
+Run the C++ unit tests after building with `--build_tests=ON`:
-An explicit `export NUM_THREADS=<n>` still wins. VS Code tasks do not read
-`~/.bashrc`, so pass `--num_threads=<n>` there.
+```bash
+cd cpp/build && ctest -V
+```
+
+## Velox static-link packaging
-## Run the tests
+The static configuration uses `apache/gluten:vcpkg-centos-9`. It opens without
+building Gluten and persists the image's vcpkg binary cache across container
+rebuilds.
-The image unpacks a Spark distribution for every supported version under
`/opt/shims`,
-which is what `spark.test.home` needs. CI runs the Spark 3.3/3.4/3.5 unit
tests on
-JDK 17:
+Build a portable Spark 3.5 jar:
```bash
-export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
-./build/mvn test -Pspark-ut -Pbackends-velox -Pspark-3.5 -Pjava-17 \
- -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \
- -DwildcardSuites=org.apache.spark.sql.GlutenSQLQuerySuite
+./dev/buildbundle-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF \
+ --spark_version=3.5
+```
+
+S3, GCS, HDFS and ABFS are disabled by default. Enable only what the jar needs:
+
+```bash
+./dev/buildbundle-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF \
+ --spark_version=3.5 --enable_s3=ON
```
-`-DwildcardSuites` takes a fully qualified class name and keeps a run to
minutes;
-without it the whole suite runs for hours. Do not add `-pl gluten-ut`: it is an
-aggregator POM, `-pl` does not pull in its children, and the build finishes in
seconds
-having run nothing. See [HowTo](./HowTo.md#3-how-to-debug-javascala) for more.
+Each enabled feature may restore or build additional vcpkg ports. vcpkg caches
+ports by ABI hash; changing the compiler, triplet or relevant port inputs can
+invalidate that cache. Native test binaries are disabled in this packaging
+workflow.
-C++ unit tests (requires `--build_tests=ON`):
+The static image uses JDK 17 by default. For Spark 4.1:
```bash
-cd cpp/build && ctest -V
+java -version # must report 17
+
+./dev/buildbundle-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF \
+ --spark_version=4.1
```
-## Static or dynamic link?
-
-"Dynamic" and "static" describe **how third-party dependencies are linked**.
Velox
-itself is archived into `libvelox.a` and linked into Gluten's shared libraries
either
-way, so `VELOX_BUILD_SHARED` only controls whether folly, gflags and glog are
built
-shared; `devcontainer.json` sets it to `ON` to match the image.
-
-| | Dynamic (`--enable_vcpkg=OFF`, the default) | Static (`--enable_vcpkg=ON`)
|
-|---|---|---|
-| Image | `apache/gluten:centos-9-jdk8` | `apache/gluten:vcpkg-centos-9` |
-| Dependencies | Pre-installed system libraries | vcpkg ports |
-| Spark distributions | `/opt/shims`, so `gluten-ut` runs | Not installed |
-| JDK | 8, with 17 added by `post-create.sh` | 17 only |
-| Resulting jar | Needs the same shared libraries | Runs on any x86-64 Linux |
-
-Static linking exists to produce **portable release jars**, which is worth
paying for
-in CI and releases but buys a developer nothing, since the jar never leaves the
-container. The vcpkg image also lacks `/opt/shims`, so the Spark unit tests
cannot
-run in it at all.
-
-vcpkg itself caches well: `apache/gluten:vcpkg-centos-9` bakes in
-`VCPKG_BINARY_SOURCES=clear;files,/var/cache/vcpkg,readwrite`, and because the
ABI
-hash is computed per port, a checkout that has moved rebuilds only the ports it
-actually changed plus their dependents. Be aware that the ABI hash also covers
the
-toolchain, so a change of compiler or triplet invalidates every port at once,
and
-vcpkg does not explain why it started rebuilding.
-
-To reproduce a static-link problem, switch `image` to
`apache/gluten:vcpkg-centos-9`
-and build with `./dev/ci-velox-buildstatic-centos-9.sh` (it exports
`NUM_THREADS=2`
-for CI runners; raise it first). Build trees are tied to the image that
produced
-them — `cpp/build/CMakeCache.txt` records the vcpkg toolchain and
-`ep/build-velox/build/velox_ep/_build/` records `VELOX_BUILD_SHARED` — so wipe
them
-when switching:
+The static image does not include `/opt/shims`; use the dynamic configuration
+for Spark unit tests. On arm64, `post-create.sh` also selects the arm64 vcpkg
+triplet and enables the required system binaries automatically.
+
+## Build parallelism
+
+`builddeps-veloxbe.sh` defaults `NUM_THREADS` to `nproc --ignore=2`, which
+ignores memory. Velox's heavier translation units use about 3.5 GB each, so a
+core-rich machine can invoke the OOM killer. `post-create.sh` exports a value
+allowing about 4 GB per job and recomputes it whenever a shell opens, so it
also
+follows a resized Codespace.
+
+An explicit `export NUM_THREADS=<n>` still wins. VS Code tasks do not read
+`~/.bashrc`, so pass `--num_threads=<n>` in a task.
+
+## Switch between configurations
+
+Static and dynamic build trees are not interchangeable:
+`cpp/build/CMakeCache.txt` records the vcpkg toolchain, and
+`ep/build-velox/build/velox_ep/_build/` records dependency linkage. Remove the
+image-specific state after switching:
```bash
rm -rf cpp/build ep/build-velox/build/velox_ep/_build ep/_ep \
dev/vcpkg/.vcpkg dev/vcpkg/vcpkg_installed
```
+The configurations use separate ccache, Maven and vcpkg-cache volumes, so
+rebuilding one environment does not contaminate the other.
+
## Machine sizing
-`hostRequirements` asks for 4 CPUs, 16 GB of memory and 64 GB of storage.
Storage is
-the binding constraint: the image plus the Velox build tree does not fit the
32 GB
-disk of the smaller Codespaces machine types, so those are marked "Below dev
container
-requirements". Only Codespaces honours `hostRequirements`; other runtimes
ignore it,
-so size the Docker VM yourself.
+The dynamic configuration requests 4 CPUs, 16 GB of memory and 64 GB of
+storage. The static configuration requests 8 CPUs, 64 GB of memory and 64 GB
+of storage because static linking needs more memory.
+
+Only Codespaces honors `hostRequirements`; for local development, size the
+Docker VM yourself.
-## Use another image
+## Images
-Other pre-built images are published to
-[Docker Hub](https://hub.docker.com/r/apache/gluten/tags); their Dockerfiles
live in
-[`dev/docker/`](https://github.com/apache/gluten/tree/main/dev/docker) and are
-described in [Velox Backend CI](./velox-backend-CI.md#docker-build).
-`apache/gluten:centos-9-jdk17` is the same dynamic-link image with JDK 17 as
the
-default; `centos-8-*` variants build against an older glibc. To switch, edit
the
-`image` field in `.devcontainer/devcontainer.json`.
+Published images are available on
+[Docker Hub](https://hub.docker.com/r/apache/gluten/tags); their Dockerfiles
+live in [`dev/docker/`](https://github.com/apache/gluten/tree/main/dev/docker)
+and are described in
+[Velox Backend CI](./velox-backend-CI.md#docker-build).
-To use these images outside a Dev Container, see
+To use the images outside a Dev Container, see
[Build Gluten Velox backend in docker](./velox-backend-build-in-docker.md).
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
