This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-8666-341076c9707730bcdfddb01fcd30823bd0b3fc18 in repository https://gitbox.apache.org/repos/asf/texera.git
commit 9ead3ac13a3f11a70457f0159c8f7a11cb2c4535 Author: ali risheh <[email protected]> AuthorDate: Fri Sep 25 18:02:48 2026 +0000 feat(mounter): let a deployment give geesefs a disk cache (#8666) ### What changes were proposed in this PR? `bin/mounter/mounter.py` starts geesefs with `--memory-limit 512` and no `--cache`, whose default is off. Nothing a mount serves is written to local disk, so every read goes back to the object store through the file service and LakeFS. Reading the same 200 MiB three times through a mount measured 179, 137 and 124 MB/s — no warming at all — against 12.4 GB/s for the same file on the node's local disk. This adds `mounter.cacheRoot`, empty by default. When set, geesefs is given `--cache <dir>`. The cache is keyed by repository and commit, not by computing unit: a commit is immutable, so every unit on a node reads one cache instead of each new unit starting cold — which matters most for a shared public unit, where the first execution pays and the rest benefit. `cacheRoot` is a separate value from `hostMountRoot` rather than a directory inside it. `hostMountRoot` holds only mount points and usually sits on the OS disk, while a cache is sized by the data being cached, so the two generally belong on different volumes. Off by default, so no deployment changes behaviour until an operator opts in. ### Any related issues, documentation, discussions? Fixes #8665. ### How was this PR tested? Two cases added to `bin/mounter/tests/test_mounter.py`: that no `--cache` is passed by default, and that with a cache root set the directory handed to geesefs is `<cacheRoot>/<repo>/<commit>` and exists. Full mounter suite passes (95 tests). The chart was rendered both ways with `mounter.enabled=true`: without `cacheRoot` the DaemonSet has no `CACHE_ROOT` env, no extra volumeMount and no extra volume; with `cacheRoot` set it has all three, pointing at the configured path. `bin/k8s/tests/test_helm_values.sh` passes. Tested with a live server. ### Was this PR authored or co-authored using generative AI tooling? Generated-by: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 (1M context) <[email protected]> --- .../templates/base/mounter/mounter-daemonset.yaml | 14 +++++++++ bin/k8s/values.yaml | 7 +++++ bin/mounter/mounter.py | 20 +++++++++++++ bin/mounter/tests/test_mounter.py | 35 ++++++++++++++++++++++ 4 files changed, 76 insertions(+) diff --git a/bin/k8s/templates/base/mounter/mounter-daemonset.yaml b/bin/k8s/templates/base/mounter/mounter-daemonset.yaml index 43927260b0..8c2fdecf66 100644 --- a/bin/k8s/templates/base/mounter/mounter-daemonset.yaml +++ b/bin/k8s/templates/base/mounter/mounter-daemonset.yaml @@ -55,6 +55,10 @@ spec: value: "{{ .Values.mounter.port }}" - name: MOUNT_ROOT value: "{{ .Values.mounter.hostMountRoot }}" + {{- if .Values.mounter.cacheRoot }} + - name: CACHE_ROOT + value: "{{ .Values.mounter.cacheRoot }}" + {{- end }} - name: POOL_NAMESPACE value: "{{ .Values.workflowComputingUnitPool.namespace }}" - name: CU_POD_NAME_PREFIX @@ -76,9 +80,19 @@ spec: - name: texera-mounts mountPath: {{ .Values.mounter.hostMountRoot }} mountPropagation: Bidirectional + {{- if .Values.mounter.cacheRoot }} + - name: texera-mount-cache + mountPath: {{ .Values.mounter.cacheRoot }} + {{- end }} volumes: - name: texera-mounts hostPath: path: {{ .Values.mounter.hostMountRoot }} type: DirectoryOrCreate + {{- if .Values.mounter.cacheRoot }} + - name: texera-mount-cache + hostPath: + path: {{ .Values.mounter.cacheRoot }} + type: DirectoryOrCreate + {{- end }} {{- end }} diff --git a/bin/k8s/values.yaml b/bin/k8s/values.yaml index 5b4a2bc1d7..a0ac347118 100644 --- a/bin/k8s/values.yaml +++ b/bin/k8s/values.yaml @@ -422,6 +422,13 @@ mounter: hostMountRoot: /var/lib/texera-mounts # Where a computing-unit pod sees its mounts. podMountRoot: /mnt/texera-mounts + # Host directory for geesefs's on-disk cache. Empty (the default) means no cache, so every + # read of a mounted repository is served from the object store. Set it when a mount is read + # repeatedly -- a reference dataset scanned on every execution -- and the node has room for + # the data being cached. Deliberately separate from hostMountRoot rather than a directory + # inside it: hostMountRoot holds only mount points and usually sits on the OS disk, while a + # cache is sized by the data, so the two generally belong on different volumes. + cacheRoot: "" serviceAccountName: texera-mounter-service-account # The mounter authenticates its callers: a caller presents a service-account token # projected for the mounter's audience, which the mounter verifies with TokenReview and diff --git a/bin/mounter/mounter.py b/bin/mounter/mounter.py index 3dc939679f..83c26d7608 100644 --- a/bin/mounter/mounter.py +++ b/bin/mounter/mounter.py @@ -50,6 +50,9 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from urllib.parse import parse_qs, urlparse MOUNT_ROOT = os.environ.get("MOUNT_ROOT", "/var/lib/texera-mounts") +# Where geesefs keeps its on-disk page cache, if a deployment gives it one. Empty means +# no cache, which is geesefs's own default: every read goes back to the object store. +CACHE_ROOT = os.environ.get("CACHE_ROOT", "").strip() MOUNTER_PORT = int(os.environ.get("MOUNTER_PORT", "8100")) POOL_NAMESPACE = os.environ.get("POOL_NAMESPACE", "texera-workflow-computing-unit-pool") MOUNT_SECRET_PLACEHOLDER = "texera-jwt-mount" @@ -237,6 +240,23 @@ def do_mount(cuid, repo, commit, jwt, file_service_base): "--endpoint", file_service_base, "--memory-limit", "512", "-o", "ro,allow_other", + ] + if CACHE_ROOT: + # CACHE_ROOT itself is never created here, only the two levels under it. The chart + # mounts it as a volume, so if it is missing the volume is missing -- and creating + # it would put the cache in the container's own writable layer, where it is lost on + # restart and can fill the node's disk with hundreds of GB nobody is watching. + if not os.path.isdir(CACHE_ROOT): + raise RuntimeError( + f"CACHE_ROOT {CACHE_ROOT!r} is not a directory; is the cache volume mounted?" + ) + # Keyed by repository and commit, not by computing unit: a commit is immutable, so + # every unit on this node reads one cache rather than each new unit starting cold. + # Both are already validated as single path segments above. + cache_dir = os.path.join(CACHE_ROOT, repo, commit) + os.makedirs(cache_dir, exist_ok=True) + cmd += ["--cache", cache_dir] + cmd += [ f"{repo}:{commit}", target, ] diff --git a/bin/mounter/tests/test_mounter.py b/bin/mounter/tests/test_mounter.py index 9e0a5321b8..c7b8447292 100644 --- a/bin/mounter/tests/test_mounter.py +++ b/bin/mounter/tests/test_mounter.py @@ -146,6 +146,41 @@ def test_do_mount_passes_the_jwt_as_the_s3_access_key(mounter): assert kwargs["env"]["AWS_ACCESS_KEY_ID"] == "the-user-jwt" +def test_do_mount_asks_for_no_cache_by_default(mounter): + mounter.do_mount("7", "dataset-1", "abc", "t", "http://file-service:9092";) + + cmd, _ = mounter.runs[0] + # geesefs's own default is no disk cache, and an unconfigured deployment keeps it. + assert "--cache" not in cmd + + +def test_do_mount_caches_by_repository_and_commit_when_a_cache_root_is_set(mounter, tmp_path, monkeypatch): + # The chart mounts CACHE_ROOT as a volume, so it is already there. + (tmp_path / "cache").mkdir() + monkeypatch.setattr(mounter, "CACHE_ROOT", str(tmp_path / "cache")) + + mounter.do_mount("7", "dataset-1", "abc", "t", "http://file-service:9092";) + + cmd, _ = mounter.runs[0] + cache_dir = cmd[cmd.index("--cache") + 1] + # Keyed by repo and commit, never by cuid: a commit is immutable, so a second computing + # unit on this node reuses the first one's cache instead of starting cold. + assert cache_dir == str(tmp_path / "cache" / "dataset-1" / "abc") + assert os.path.isdir(cache_dir) + + +def test_do_mount_refuses_a_cache_root_that_is_not_there(mounter, tmp_path, monkeypatch): + absent = tmp_path / "not-mounted" + monkeypatch.setattr(mounter, "CACHE_ROOT", str(absent)) + + # Creating it would cache into the container's own filesystem instead of the volume. + with pytest.raises(RuntimeError): + mounter.do_mount("7", "dataset-1", "abc", "t", "http://file-service:9092";) + + assert not absent.exists() + assert mounter.runs == [] + + def test_do_mount_is_idempotent_for_a_live_mount(mounter, cu_dir): _, target = cu_dir("7", commit="abc") mounter.set_mounts(target)
