This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 5ab0389b1 fix(io): treat ADLS Gen2 / OneLake directory blobs as
directories in blobfs WalkDir (#2180)
5ab0389b1 is described below
commit 5ab0389b1ecb1dd1a508e921909d3e2bcffb26b4
Author: Ajitesh Singh <[email protected]>
AuthorDate: Tue Oct 13 05:34:18 2026 -0700
fix(io): treat ADLS Gen2 / OneLake directory blobs as directories in blobfs
WalkDir (#2180)
* fix(io): walk hierarchical namespace directories in blobfs WalkDir
On ADLS Gen2 accounts with a hierarchical namespace (such as OneLake),
every directory is also stored as a 0-byte blob named after it, with no
trailing slash. gocloud's fs.FS treats any key with an exact object as a
file, so WalkDir stopped at the walk root and reported it as the only
file. Orphan cleanup then listed the table location as its only orphan.
WalkDir now walks a small adapter over the bucket. A key with objects
under it is a directory, and a 0-byte blob named after a child directory
is hidden from listings. Trailing-slash markers on flat stores behave as
before, and the walk makes one listing per directory instead of an
existence check plus a listing.
Fixes #2159
Co-Authored-By: Claude Opus 5.5 <[email protected]>
* docs(io): shorten walkFS comments and note empty HNS directories
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---------
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
io/gocloud/blobfs/blob.go | 126 ++++++++++++++++++++++++++++++++++++++++-
io/gocloud/blobfs/blob_test.go | 45 +++++++++++++++
2 files changed, 170 insertions(+), 1 deletion(-)
diff --git a/io/gocloud/blobfs/blob.go b/io/gocloud/blobfs/blob.go
index 0bdad018b..5988870aa 100644
--- a/io/gocloud/blobfs/blob.go
+++ b/io/gocloud/blobfs/blob.go
@@ -429,6 +429,130 @@ func isDirectoryMarker(walkRootKey string, dirEntry
fs.DirEntry) bool {
return ok && obj.Key == directoryMarker(walkRootKey)
}
+// walkFS walks a bucket for fs.WalkDir, treating any key with children as a
directory,
+// so the 0-byte directory blobs of ADLS hierarchical namespaces (OneLake)
aren't files.
+type walkFS struct{ bfs *FileIO }
+
+var (
+ _ fs.StatFS = walkFS{}
+ _ fs.ReadDirFS = walkFS{}
+)
+
+func (w walkFS) Open(name string) (fs.File, error) { return
w.bfs.Bucket.Open(name) }
+
+func (w walkFS) Stat(name string) (fs.FileInfo, error) {
+ if !fs.ValidPath(name) {
+ return nil, &fs.PathError{Op: "stat", Path: name, Err:
fs.ErrInvalid}
+ }
+
+ if name == "." {
+ return walkDirEntry{name: "."}, nil
+ }
+
+ // One child is enough to make name a directory, so fetch a single key.
+ children, _, err := w.bfs.ListPage(w.bfs.ctx, blob.FirstPageToken, 1,
&blob.ListOptions{Prefix: name + "/"})
+ if err != nil {
+ return nil, blobErrToFsErr("stat", name, err)
+ }
+ if len(children) > 0 {
+ return walkDirEntry{name: pathpkg.Base(name)}, nil
+ }
+
+ attrs, err := w.bfs.Attributes(w.bfs.ctx, name)
+ if err != nil {
+ return nil, blobErrToFsErr("stat", name, err)
+ }
+
+ return blobFileInfo{
+ name: pathpkg.Base(name),
+ size: attrs.Size,
+ mode: fs.ModeIrregular,
+ modTime: attrs.ModTime,
+ sys: attrs,
+ }, nil
+}
+
+func (w walkFS) ReadDir(name string) ([]fs.DirEntry, error) {
+ if !fs.ValidPath(name) {
+ return nil, &fs.PathError{Op: "readdir", Path: name, Err:
fs.ErrInvalid}
+ }
+
+ prefix := ""
+ if name != "." {
+ prefix = name + "/"
+ }
+
+ var files []*blob.ListObject
+ dirs := map[string]bool{}
+ iter := w.bfs.List(&blob.ListOptions{Prefix: prefix, Delimiter: "/"})
+ for {
+ obj, err := iter.Next(w.bfs.ctx)
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return nil, blobErrToFsErr("readdir", name, err)
+ }
+
+ if obj.IsDir {
+ dirs[strings.TrimSuffix(strings.TrimPrefix(obj.Key,
prefix), "/")] = true
+ } else {
+ files = append(files, obj)
+ }
+ }
+
+ entries := make([]fs.DirEntry, 0, len(dirs)+len(files))
+ for child := range dirs {
+ entries = append(entries, walkDirEntry{name: child})
+ }
+ for _, obj := range files {
+ child := strings.TrimPrefix(obj.Key, prefix)
+ if obj.Size == 0 && dirs[child] {
+ // A hierarchical-namespace blob for a child directory.
An empty directory's blob
+ // can't be told apart without hdi_isfolder metadata,
so it stays a 0-byte file.
+ continue
+ }
+
+ // Name entries the way gocloud's fs.FS does, so a "dir/"
marker keeps
+ // the name it had before walkFS.
+ entries = append(entries, walkFileEntry{blobFileInfo{
+ name: pathpkg.Base(obj.Key),
+ size: obj.Size,
+ mode: fs.ModeIrregular,
+ modTime: obj.ModTime,
+ sys: obj,
+ }})
+ }
+
+ if len(entries) == 0 && name != "." {
+ return nil, &fs.PathError{Op: "readdir", Path: name, Err:
fs.ErrNotExist}
+ }
+
+ slices.SortFunc(entries, func(a, b fs.DirEntry) int { return
strings.Compare(a.Name(), b.Name()) })
+
+ return entries, nil
+}
+
+// walkDirEntry is a directory found by walkFS. It implements fs.FileInfo and
+// fs.DirEntry.
+type walkDirEntry struct{ name string }
+
+func (d walkDirEntry) Name() string { return d.name }
+func (d walkDirEntry) Size() int64 { return 0 }
+func (d walkDirEntry) Mode() fs.FileMode { return fs.ModeDir }
+func (d walkDirEntry) Type() fs.FileMode { return fs.ModeDir }
+func (d walkDirEntry) ModTime() time.Time { return time.Time{} }
+func (d walkDirEntry) IsDir() bool { return true }
+func (d walkDirEntry) Sys() any { return nil }
+func (d walkDirEntry) Info() (fs.FileInfo, error) { return d, nil }
+
+// walkFileEntry is an object found by walkFS. Its Info().Sys() is the
+// *blob.ListObject it was listed from.
+type walkFileEntry struct{ blobFileInfo }
+
+func (f walkFileEntry) Type() fs.FileMode { return f.mode.Type() }
+func (f walkFileEntry) Info() (fs.FileInfo, error) { return f.blobFileInfo,
nil }
+
func (bfs *FileIO) WalkDir(root string, fn fs.WalkDirFunc) error {
location, err := bfs.resolveLocation(root)
var walkPath string
@@ -442,7 +566,7 @@ func (bfs *FileIO) WalkDir(root string, fn fs.WalkDirFunc)
error {
walkPath = location.key
}
- return fs.WalkDir(bfs.Bucket, walkPath, func(path string, d
fs.DirEntry, err error) error {
+ return fs.WalkDir(walkFS{bfs}, walkPath, func(path string, d
fs.DirEntry, err error) error {
// Skip the marker for the path passed to WalkDir. Otherwise
walking
// "warehouse/ns" reports "warehouse/ns/" as a child file.
if err == nil && isDirectoryMarker(walkPath, d) {
diff --git a/io/gocloud/blobfs/blob_test.go b/io/gocloud/blobfs/blob_test.go
index cad387779..125a2f042 100644
--- a/io/gocloud/blobfs/blob_test.go
+++ b/io/gocloud/blobfs/blob_test.go
@@ -551,6 +551,51 @@ func TestBlobFileIOWalkDirSubPath(t *testing.T) {
assert.ElementsMatch(t, expected, walked)
}
+func TestBlobFileIOWalkDirHierarchicalNamespace(t *testing.T) {
+ ctx := context.Background()
+
+ bucket := memblob.OpenBucket(nil)
+ defer bucket.Close()
+
+ // ADLS Gen2 hierarchical namespaces store each directory as a 0-byte
+ // blob named after it, with no trailing slash.
+ for _, k := range []string{"tbl", "tbl/data", "tbl/metadata"} {
+ require.NoError(t, bucket.WriteAll(ctx, k, nil, nil))
+ }
+ require.NoError(t, bucket.WriteAll(ctx, "tbl/data/a.parquet",
[]byte("a"), nil))
+ require.NoError(t, bucket.WriteAll(ctx,
"tbl/metadata/v1.metadata.json", []byte("b"), nil))
+ require.NoError(t, bucket.WriteAll(ctx, "tbl/empty.txt", nil, nil))
+
+ bfs := testBlobFileIO(ctx, "mybucket", bucket)
+
+ var dirs, files []string
+ err := bfs.WalkDir("s3://mybucket/tbl", func(path string, d
fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+
+ if d.IsDir() {
+ dirs = append(dirs, path)
+ } else {
+ files = append(files, path)
+ }
+
+ return nil
+ })
+ require.NoError(t, err)
+
+ assert.Equal(t, []string{
+ "s3://mybucket/tbl",
+ "s3://mybucket/tbl/data",
+ "s3://mybucket/tbl/metadata",
+ }, dirs)
+ assert.Equal(t, []string{
+ "s3://mybucket/tbl/data/a.parquet",
+ "s3://mybucket/tbl/empty.txt",
+ "s3://mybucket/tbl/metadata/v1.metadata.json",
+ }, files)
+}
+
func TestBlobFileIORawQueryFragmentRoundTrip(t *testing.T) {
ctx := context.Background()
