This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/texera.git
The following commit(s) were added to refs/heads/main by this push:
new 9ead3ac13a feat(mounter): let a deployment give geesefs a disk cache
(#8666)
9ead3ac13a is described below
commit 9ead3ac13a3f11a70457f0159c8f7a11cb2c4535
Author: ali risheh <[email protected]>
AuthorDate: Fri Sep 25 18:02:48 2026 +0000
feat(mounter): let a deployment give geesefs a disk cache (#8666)
### What changes were proposed in this PR?
`bin/mounter/mounter.py` starts geesefs with `--memory-limit 512` and no
`--cache`, whose default is off. Nothing a mount serves is written to
local disk, so every read goes back to the object store through the file
service and LakeFS. Reading the same 200 MiB three times through a mount
measured 179, 137 and 124 MB/s — no warming at all — against 12.4 GB/s
for the same file on the node's local disk.
This adds `mounter.cacheRoot`, empty by default. When set, geesefs is
given `--cache <dir>`.
The cache is keyed by repository and commit, not by computing unit: a
commit is immutable, so every unit on a node reads one cache instead of
each new unit starting cold — which matters most for a shared public
unit, where the first execution pays and the rest benefit.
`cacheRoot` is a separate value from `hostMountRoot` rather than a
directory inside it. `hostMountRoot` holds only mount points and usually
sits on the OS disk, while a cache is sized by the data being cached, so
the two generally belong on different volumes.
Off by default, so no deployment changes behaviour until an operator
opts in.
### Any related issues, documentation, discussions?
Fixes #8665.
### How was this PR tested?
Two cases added to `bin/mounter/tests/test_mounter.py`: that no
`--cache` is passed by default, and that with a cache root set the
directory handed to geesefs is `<cacheRoot>/<repo>/<commit>` and exists.
Full mounter suite passes (95 tests).
The chart was rendered both ways with `mounter.enabled=true`: without
`cacheRoot` the DaemonSet has no `CACHE_ROOT` env, no extra volumeMount
and no extra volume; with `cacheRoot` set it has all three, pointing at
the configured path. `bin/k8s/tests/test_helm_values.sh` passes.
Tested with a live server.
### Was this PR authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5
---------
Co-authored-by: Claude Opus 5 (1M context) <[email protected]>
---
.../templates/base/mounter/mounter-daemonset.yaml | 14 +++++++++
bin/k8s/values.yaml | 7 +++++
bin/mounter/mounter.py | 20 +++++++++++++
bin/mounter/tests/test_mounter.py | 35 ++++++++++++++++++++++
4 files changed, 76 insertions(+)
diff --git a/bin/k8s/templates/base/mounter/mounter-daemonset.yaml
b/bin/k8s/templates/base/mounter/mounter-daemonset.yaml
index 43927260b0..8c2fdecf66 100644
--- a/bin/k8s/templates/base/mounter/mounter-daemonset.yaml
+++ b/bin/k8s/templates/base/mounter/mounter-daemonset.yaml
@@ -55,6 +55,10 @@ spec:
value: "{{ .Values.mounter.port }}"
- name: MOUNT_ROOT
value: "{{ .Values.mounter.hostMountRoot }}"
+ {{- if .Values.mounter.cacheRoot }}
+ - name: CACHE_ROOT
+ value: "{{ .Values.mounter.cacheRoot }}"
+ {{- end }}
- name: POOL_NAMESPACE
value: "{{ .Values.workflowComputingUnitPool.namespace }}"
- name: CU_POD_NAME_PREFIX
@@ -76,9 +80,19 @@ spec:
- name: texera-mounts
mountPath: {{ .Values.mounter.hostMountRoot }}
mountPropagation: Bidirectional
+ {{- if .Values.mounter.cacheRoot }}
+ - name: texera-mount-cache
+ mountPath: {{ .Values.mounter.cacheRoot }}
+ {{- end }}
volumes:
- name: texera-mounts
hostPath:
path: {{ .Values.mounter.hostMountRoot }}
type: DirectoryOrCreate
+ {{- if .Values.mounter.cacheRoot }}
+ - name: texera-mount-cache
+ hostPath:
+ path: {{ .Values.mounter.cacheRoot }}
+ type: DirectoryOrCreate
+ {{- end }}
{{- end }}
diff --git a/bin/k8s/values.yaml b/bin/k8s/values.yaml
index 5b4a2bc1d7..a0ac347118 100644
--- a/bin/k8s/values.yaml
+++ b/bin/k8s/values.yaml
@@ -422,6 +422,13 @@ mounter:
hostMountRoot: /var/lib/texera-mounts
# Where a computing-unit pod sees its mounts.
podMountRoot: /mnt/texera-mounts
+ # Host directory for geesefs's on-disk cache. Empty (the default) means no
cache, so every
+ # read of a mounted repository is served from the object store. Set it when
a mount is read
+ # repeatedly -- a reference dataset scanned on every execution -- and the
node has room for
+ # the data being cached. Deliberately separate from hostMountRoot rather
than a directory
+ # inside it: hostMountRoot holds only mount points and usually sits on the
OS disk, while a
+ # cache is sized by the data, so the two generally belong on different
volumes.
+ cacheRoot: ""
serviceAccountName: texera-mounter-service-account
# The mounter authenticates its callers: a caller presents a service-account
token
# projected for the mounter's audience, which the mounter verifies with
TokenReview and
diff --git a/bin/mounter/mounter.py b/bin/mounter/mounter.py
index 3dc939679f..83c26d7608 100644
--- a/bin/mounter/mounter.py
+++ b/bin/mounter/mounter.py
@@ -50,6 +50,9 @@ from http.server import BaseHTTPRequestHandler,
ThreadingHTTPServer
from urllib.parse import parse_qs, urlparse
MOUNT_ROOT = os.environ.get("MOUNT_ROOT", "/var/lib/texera-mounts")
+# Where geesefs keeps its on-disk page cache, if a deployment gives it one.
Empty means
+# no cache, which is geesefs's own default: every read goes back to the object
store.
+CACHE_ROOT = os.environ.get("CACHE_ROOT", "").strip()
MOUNTER_PORT = int(os.environ.get("MOUNTER_PORT", "8100"))
POOL_NAMESPACE = os.environ.get("POOL_NAMESPACE",
"texera-workflow-computing-unit-pool")
MOUNT_SECRET_PLACEHOLDER = "texera-jwt-mount"
@@ -237,6 +240,23 @@ def do_mount(cuid, repo, commit, jwt, file_service_base):
"--endpoint", file_service_base,
"--memory-limit", "512",
"-o", "ro,allow_other",
+ ]
+ if CACHE_ROOT:
+ # CACHE_ROOT itself is never created here, only the two levels under
it. The chart
+ # mounts it as a volume, so if it is missing the volume is missing --
and creating
+ # it would put the cache in the container's own writable layer, where
it is lost on
+ # restart and can fill the node's disk with hundreds of GB nobody is
watching.
+ if not os.path.isdir(CACHE_ROOT):
+ raise RuntimeError(
+ f"CACHE_ROOT {CACHE_ROOT!r} is not a directory; is the cache
volume mounted?"
+ )
+ # Keyed by repository and commit, not by computing unit: a commit is
immutable, so
+ # every unit on this node reads one cache rather than each new unit
starting cold.
+ # Both are already validated as single path segments above.
+ cache_dir = os.path.join(CACHE_ROOT, repo, commit)
+ os.makedirs(cache_dir, exist_ok=True)
+ cmd += ["--cache", cache_dir]
+ cmd += [
f"{repo}:{commit}",
target,
]
diff --git a/bin/mounter/tests/test_mounter.py
b/bin/mounter/tests/test_mounter.py
index 9e0a5321b8..c7b8447292 100644
--- a/bin/mounter/tests/test_mounter.py
+++ b/bin/mounter/tests/test_mounter.py
@@ -146,6 +146,41 @@ def
test_do_mount_passes_the_jwt_as_the_s3_access_key(mounter):
assert kwargs["env"]["AWS_ACCESS_KEY_ID"] == "the-user-jwt"
+def test_do_mount_asks_for_no_cache_by_default(mounter):
+ mounter.do_mount("7", "dataset-1", "abc", "t", "http://file-service:9092";)
+
+ cmd, _ = mounter.runs[0]
+ # geesefs's own default is no disk cache, and an unconfigured deployment
keeps it.
+ assert "--cache" not in cmd
+
+
+def
test_do_mount_caches_by_repository_and_commit_when_a_cache_root_is_set(mounter,
tmp_path, monkeypatch):
+ # The chart mounts CACHE_ROOT as a volume, so it is already there.
+ (tmp_path / "cache").mkdir()
+ monkeypatch.setattr(mounter, "CACHE_ROOT", str(tmp_path / "cache"))
+
+ mounter.do_mount("7", "dataset-1", "abc", "t", "http://file-service:9092";)
+
+ cmd, _ = mounter.runs[0]
+ cache_dir = cmd[cmd.index("--cache") + 1]
+ # Keyed by repo and commit, never by cuid: a commit is immutable, so a
second computing
+ # unit on this node reuses the first one's cache instead of starting cold.
+ assert cache_dir == str(tmp_path / "cache" / "dataset-1" / "abc")
+ assert os.path.isdir(cache_dir)
+
+
+def test_do_mount_refuses_a_cache_root_that_is_not_there(mounter, tmp_path,
monkeypatch):
+ absent = tmp_path / "not-mounted"
+ monkeypatch.setattr(mounter, "CACHE_ROOT", str(absent))
+
+ # Creating it would cache into the container's own filesystem instead of
the volume.
+ with pytest.raises(RuntimeError):
+ mounter.do_mount("7", "dataset-1", "abc", "t",
"http://file-service:9092";)
+
+ assert not absent.exists()
+ assert mounter.runs == []
+
+
def test_do_mount_is_idempotent_for_a_live_mount(mounter, cu_dir):
_, target = cu_dir("7", commit="abc")
mounter.set_mounts(target)
