This is an automated email from the ASF dual-hosted git repository.
SaketaChalamchala pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 77ceb2969b5 HDDS-15624. Resolve link bucket properties in listBucket.
(#10745)
77ceb2969b5 is described below
commit 77ceb2969b507096fe97482da3a1e693ccc64561
Author: SaketaChalamchala <[email protected]>
AuthorDate: Thu Jul 23 10:25:22 2026 -0700
HDDS-15624. Resolve link bucket properties in listBucket. (#10745)
Co-authored-by: Wei-Chiu Chuang <[email protected]>
Co-authored-by: Copilot Autofix powered by AI
<[email protected]>
Co-authored-by: Wei-Chiu Chuang <[email protected]>
Co-authored-by: Cursor <[email protected]>
---
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 25 +++++
.../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 50 ++++++++++
.../org/apache/hadoop/ozone/om/OzoneManager.java | 110 +++++++++++++--------
.../hadoop/ozone/om/TestBucketManagerImpl.java | 80 +++++++++++++++
4 files changed, 224 insertions(+), 41 deletions(-)
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java
index 8da2be2755a..463b9de0d95 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java
@@ -396,6 +396,31 @@ public Builder toBuilder() {
.setTags(tags);
}
+ /**
+ * Returns a copy of this bucket with operational properties taken from
+ * {@code source}. Link identity fields (volume, name, owner, source path,
+ * ACLs, timestamps, object/update IDs) are unchanged.
+ *
+ * <p>When adding new operational bucket fields, update this method if they
+ * should be resolved from a link's source bucket.
+ */
+ public OmBucketInfo withOperationalPropertiesFrom(OmBucketInfo source) {
+ return toBuilder()
+ .setDefaultReplicationConfig(source.getDefaultReplicationConfig())
+ .setIsVersionEnabled(source.getIsVersionEnabled())
+ .setStorageType(source.getStorageType())
+ .setQuotaInBytes(source.getQuotaInBytes())
+ .setQuotaInNamespace(source.getQuotaInNamespace())
+ .setUsedBytes(source.getUsedBytes())
+ .setUsedNamespace(source.getUsedNamespace())
+ .setSnapshotUsedBytes(source.getSnapshotUsedBytes())
+ .setSnapshotUsedNamespace(source.getSnapshotUsedNamespace())
+ .addAllMetadata(source.getMetadata())
+ .setBucketLayout(source.getBucketLayout())
+ .setTags(source.getTags())
+ .build();
+ }
+
/**
* Builder for OmBucketInfo.
*/
diff --git
a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketInfo.java
b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketInfo.java
index 857103a20c0..5f816ff4c20 100644
---
a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketInfo.java
+++
b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketInfo.java
@@ -97,6 +97,56 @@ public void testClone() {
cloneBucketInfo.getAcls().get(0));
}
+ @Test
+ public void testWithOperationalPropertiesFromPreservesLinkIdentity() {
+ ECReplicationConfig sourceReplication = new ECReplicationConfig(3, 2);
+ OmBucketInfo source = OmBucketInfo.newBuilder()
+ .setVolumeName("vol1")
+ .setBucketName("source")
+ .setBucketLayout(BucketLayout.OBJECT_STORE)
+ .setStorageType(StorageType.SSD)
+ .setIsVersionEnabled(true)
+ .setQuotaInBytes(1000)
+ .setQuotaInNamespace(10)
+ .setUsedBytes(500)
+ .setUsedNamespace(5)
+ .setDefaultReplicationConfig(new
DefaultReplicationConfig(sourceReplication))
+ .addAllMetadata(Collections.singletonMap("sourceKey", "sourceValue"))
+ .build();
+
+ OmBucketInfo link = OmBucketInfo.newBuilder()
+ .setVolumeName("vol1")
+ .setBucketName("link")
+ .setSourceVolume("vol1")
+ .setSourceBucket("source")
+ .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED)
+ .setCreationTime(123L)
+ .setModificationTime(456L)
+ .addAllMetadata(Collections.singletonMap("linkKey", "linkValue"))
+ .build();
+
+ OmBucketInfo resolvedLink = link.withOperationalPropertiesFrom(source);
+
+ assertEquals("link", resolvedLink.getBucketName());
+ assertEquals("vol1", resolvedLink.getVolumeName());
+ assertEquals("vol1", resolvedLink.getSourceVolume());
+ assertEquals("source", resolvedLink.getSourceBucket());
+ assertEquals(123L, resolvedLink.getCreationTime());
+ assertEquals(456L, resolvedLink.getModificationTime());
+
+ assertEquals(BucketLayout.OBJECT_STORE, resolvedLink.getBucketLayout());
+ assertEquals(StorageType.SSD, resolvedLink.getStorageType());
+ assertTrue(resolvedLink.getIsVersionEnabled());
+ assertEquals(1000, resolvedLink.getQuotaInBytes());
+ assertEquals(10, resolvedLink.getQuotaInNamespace());
+ assertEquals(500, resolvedLink.getUsedBytes());
+ assertEquals(5, resolvedLink.getUsedNamespace());
+ assertEquals(sourceReplication,
+ resolvedLink.getDefaultReplicationConfig().getReplicationConfig());
+ assertEquals("sourceValue", resolvedLink.getMetadata().get("sourceKey"));
+ assertEquals("linkValue", resolvedLink.getMetadata().get("linkKey"));
+ }
+
@Test
public void getProtobufMessageEC() {
OmBucketInfo omBucketInfo =
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index b9183831b89..709c59da16b 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -3005,8 +3005,18 @@ public List<OmBucketInfo> listBuckets(String volumeName,
String startKey,
volumeName, null, null);
}
metrics.incNumBucketLists();
- return bucketManager.listBuckets(volumeName,
+ List<OmBucketInfo> buckets = bucketManager.listBuckets(volumeName,
startKey, prefix, maxNumOfBuckets, hasSnapshot);
+ Map<Pair<String, String>, OmBucketInfo> resolvedSourceCache = new
HashMap<>();
+ for (int i = 0; i < buckets.size(); i++) {
+ try {
+ buckets.set(i, enrichLinkBucketInfo(buckets.get(i),
resolvedSourceCache));
+ } catch (IOException e) {
+ LOG.debug("Failed to enrich listBuckets entry for {}/{}; returning
raw entry",
+ volumeName, buckets.get(i).getBucketName(), e);
+ }
+ }
+ return buckets;
} catch (IOException ex) {
metrics.incNumBucketListFails();
auditSuccess = false;
@@ -3021,6 +3031,62 @@ public List<OmBucketInfo> listBuckets(String volumeName,
String startKey,
}
}
+ /**
+ * For link buckets, follows the link chain and overlays the source bucket's
+ * operational properties onto the link's {@link OmBucketInfo}. Non-link
+ * buckets and dangling links are returned unchanged.
+ */
+ private OmBucketInfo enrichLinkBucketInfo(OmBucketInfo bucketInfo)
+ throws IOException {
+ return enrichLinkBucketInfo(bucketInfo, null);
+ }
+
+ private OmBucketInfo enrichLinkBucketInfo(
+ OmBucketInfo bucketInfo,
+ Map<Pair<String, String>, OmBucketInfo> resolvedSourceCache)
+ throws IOException {
+ if (!bucketInfo.isLink()) {
+ return bucketInfo;
+ }
+ // We already know that `bucketInfo` is a linked one,
+ // so we skip one `getBucketInfo` and start with the known link.
+ ResolvedBucket resolvedBucket =
+ resolveBucketLink(Pair.of(
+ bucketInfo.getSourceVolume(),
+ bucketInfo.getSourceBucket()),
+ true);
+
+ // If it is a dangling link it means no real bucket exists,
+ // for example, it could have been deleted, but the links still present.
+ if (resolvedBucket.isDangling()) {
+ return bucketInfo;
+ }
+ OmBucketInfo realBucket = getResolvedSourceBucket(resolvedBucket,
resolvedSourceCache);
+ return bucketInfo.withOperationalPropertiesFrom(realBucket);
+ }
+
+ private OmBucketInfo getResolvedSourceBucket(
+ ResolvedBucket resolvedBucket,
+ Map<Pair<String, String>, OmBucketInfo> resolvedSourceCache)
+ throws IOException {
+ Pair<String, String> sourceKey = Pair.of(
+ resolvedBucket.realVolume(),
+ resolvedBucket.realBucket());
+ if (resolvedSourceCache != null) {
+ OmBucketInfo cachedSource = resolvedSourceCache.get(sourceKey);
+ if (cachedSource != null) {
+ return cachedSource;
+ }
+ }
+ OmBucketInfo realBucket = bucketManager.getBucketInfo(
+ resolvedBucket.realVolume(),
+ resolvedBucket.realBucket());
+ if (resolvedSourceCache != null) {
+ resolvedSourceCache.put(sourceKey, realBucket);
+ }
+ return realBucket;
+ }
+
/**
* Gets the bucket information.
*
@@ -3042,46 +3108,8 @@ public OmBucketInfo getBucketInfo(String volume, String
bucket)
}
metrics.incNumBucketInfos();
- OmBucketInfo bucketInfo = bucketManager.getBucketInfo(volume, bucket);
-
- // No links - return the bucket info right away.
- if (!bucketInfo.isLink()) {
- return bucketInfo;
- }
- // Otherwise follow the links to find the real bucket.
- // We already know that `bucketInfo` is a linked one,
- // so we skip one `getBucketInfo` and start with the known link.
- ResolvedBucket resolvedBucket =
- resolveBucketLink(Pair.of(
- bucketInfo.getSourceVolume(),
- bucketInfo.getSourceBucket()),
- true);
-
- // If it is a dangling link it means no real bucket exists,
- // for example, it could have been deleted, but the links still present.
- if (!resolvedBucket.isDangling()) {
- OmBucketInfo realBucket =
- bucketManager.getBucketInfo(
- resolvedBucket.realVolume(),
- resolvedBucket.realBucket());
- // Pass the real bucket metadata in the link bucket info.
- return bucketInfo.toBuilder()
- .setDefaultReplicationConfig(
- realBucket.getDefaultReplicationConfig())
- .setIsVersionEnabled(realBucket.getIsVersionEnabled())
- .setStorageType(realBucket.getStorageType())
- .setQuotaInBytes(realBucket.getQuotaInBytes())
- .setQuotaInNamespace(realBucket.getQuotaInNamespace())
- .setUsedBytes(realBucket.getUsedBytes())
- .setSnapshotUsedBytes(realBucket.getSnapshotUsedBytes())
- .setSnapshotUsedNamespace(realBucket.getSnapshotUsedNamespace())
- .setUsedNamespace(realBucket.getUsedNamespace())
- .addAllMetadata(realBucket.getMetadata())
- .setBucketLayout(realBucket.getBucketLayout())
- .build();
- }
- // If no real bucket exists, return the requested one's info.
- return bucketInfo;
+ return enrichLinkBucketInfo(
+ bucketManager.getBucketInfo(volume, bucket));
} catch (Exception ex) {
metrics.incNumBucketInfoFails();
auditSuccess = false;
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketManagerImpl.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketManagerImpl.java
index 42d748607ac..1755fb433c4 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketManagerImpl.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketManagerImpl.java
@@ -31,10 +31,12 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.List;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.client.StandaloneReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.StorageType;
@@ -460,4 +462,82 @@ public void testLinkedBucketResolution() throws Exception {
bucketInfo.getIsVersionEnabled(),
storedLinkBucket.getIsVersionEnabled());
}
+
+ @Test
+ public void testListBucketsResolvesFsoAndObsLinkLayouts() throws Exception {
+ String volume = volumeName();
+ createSampleVol(volume);
+
+ ECReplicationConfig fsoReplication = new ECReplicationConfig(3, 2);
+ OmBucketInfo fsoSource = OmBucketInfo.newBuilder()
+ .setVolumeName(volume)
+ .setBucketName("fso-source")
+ .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED)
+ .setDefaultReplicationConfig(new
DefaultReplicationConfig(fsoReplication))
+ .build();
+ writeClient.createBucket(fsoSource);
+
+ RatisReplicationConfig obsReplication =
+ RatisReplicationConfig.getInstance(ReplicationFactor.THREE);
+ OmBucketInfo obsSource = OmBucketInfo.newBuilder()
+ .setVolumeName(volume)
+ .setBucketName("obs-source")
+ .setBucketLayout(BucketLayout.OBJECT_STORE)
+ .setDefaultReplicationConfig(new
DefaultReplicationConfig(obsReplication))
+ .build();
+ writeClient.createBucket(obsSource);
+
+ OmBucketInfo fsoLink = OmBucketInfo.newBuilder()
+ .setVolumeName(volume)
+ .setBucketName("link-fso")
+ .setSourceVolume(volume)
+ .setSourceBucket("fso-source")
+ .build();
+ writeClient.createBucket(fsoLink);
+
+ OmBucketInfo obsLink = OmBucketInfo.newBuilder()
+ .setVolumeName(volume)
+ .setBucketName("link-obs")
+ .setSourceVolume(volume)
+ .setSourceBucket("obs-source")
+ .build();
+ writeClient.createBucket(obsLink);
+
+ OmBucketInfo fsoLinkInfo = writeClient.getBucketInfo(volume, "link-fso");
+ assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED,
fsoLinkInfo.getBucketLayout());
+ assertEquals(fsoReplication,
+ fsoLinkInfo.getDefaultReplicationConfig().getReplicationConfig());
+
+ OmBucketInfo obsLinkInfo = writeClient.getBucketInfo(volume, "link-obs");
+ assertEquals(BucketLayout.OBJECT_STORE, obsLinkInfo.getBucketLayout());
+ assertEquals(obsReplication,
+ obsLinkInfo.getDefaultReplicationConfig().getReplicationConfig());
+
+ List<OmBucketInfo> listedBuckets =
+ writeClient.listBuckets(volume, "", "", 100, false);
+
+ OmBucketInfo listedFsoLink = null;
+ OmBucketInfo listedObsLink = null;
+ for (OmBucketInfo listedBucket : listedBuckets) {
+ if ("link-fso".equals(listedBucket.getBucketName())) {
+ listedFsoLink = listedBucket;
+ } else if ("link-obs".equals(listedBucket.getBucketName())) {
+ listedObsLink = listedBucket;
+ }
+ }
+ assertNotNull(listedFsoLink, "link-fso not found in listBuckets response");
+ assertNotNull(listedObsLink, "link-obs not found in listBuckets response");
+
+ assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED,
listedFsoLink.getBucketLayout());
+ assertEquals(fsoReplication,
+ listedFsoLink.getDefaultReplicationConfig().getReplicationConfig());
+ assertEquals(volume, listedFsoLink.getSourceVolume());
+ assertEquals("fso-source", listedFsoLink.getSourceBucket());
+
+ assertEquals(BucketLayout.OBJECT_STORE, listedObsLink.getBucketLayout());
+ assertEquals(obsReplication,
+ listedObsLink.getDefaultReplicationConfig().getReplicationConfig());
+ assertEquals(volume, listedObsLink.getSourceVolume());
+ assertEquals("obs-source", listedObsLink.getSourceBucket());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
