This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new fcbf3c59a6 [#9736] feat(iceberg-rest): Support freshness-aware table
loading with ETag (#10498)
fcbf3c59a6 is described below
commit fcbf3c59a6f9607682a81c7f3623d582171d1574
Author: akshay thorat <[email protected]>
AuthorDate: Tue Mar 24 04:56:54 2026 -0700
[#9736] feat(iceberg-rest): Support freshness-aware table loading with ETag
(#10498)
Implement HTTP ETag-based conditional request support for the Iceberg
REST server's loadTable endpoint to enable freshness-aware caching.
When loading a table, the server now generates an ETag header from a
SHA-256 hash of the table's metadata file location. If the client sends
an If-None-Match header matching the current ETag, the server responds
with HTTP 304 Not Modified instead of the full table metadata payload.
This allows compute engines that cache Iceberg metadata (e.g., Impala)
to skip re-fetching unchanged metadata, improving read-heavy workload
performance.
Changes:
- Add If-None-Match header parameter to IcebergTableOperations.loadTable
- Generate ETag from metadata file location via SHA-256 hash
- Return 304 Not Modified when client ETag matches, 200 with ETag
otherwise
- Add tests: ETag presence, 304 on match, 200 on mismatch, ETag changes
after update, and ETag consistency across loads
Closes: #9736
---
.../service/rest/IcebergTableOperations.java | 116 +++++++++++++-
.../service/rest/TestIcebergTableOperations.java | 171 +++++++++++++++++++++
2 files changed, 283 insertions(+), 4 deletions(-)
diff --git
a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java
b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java
index e15de6d51b..9d3092503d 100644
---
a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java
+++
b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java
@@ -23,6 +23,9 @@ import com.codahale.metrics.annotation.Timed;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
import javax.inject.Inject;
@@ -40,6 +43,7 @@ import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
+import javax.ws.rs.core.EntityTag;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.apache.commons.lang3.StringUtils;
@@ -86,6 +90,8 @@ public class IcebergTableOperations {
@VisibleForTesting
public static final String X_ICEBERG_ACCESS_DELEGATION =
"X-Iceberg-Access-Delegation";
+ @VisibleForTesting public static final String IF_NONE_MATCH =
"If-None-Match";
+
private IcebergMetricsManager icebergMetricsManager;
private ObjectMapper icebergObjectMapper;
@@ -173,7 +179,7 @@ public class IcebergTableOperations {
new IcebergRequestContext(httpServletRequest(), catalogName,
isCredentialVending);
LoadTableResponse loadTableResponse =
tableOperationDispatcher.createTable(context, icebergNS,
createTableRequest);
- return IcebergRESTUtils.ok(loadTableResponse);
+ return buildResponseWithETag(loadTableResponse);
});
} catch (Exception e) {
return IcebergExceptionMapper.toRESTResponse(e);
@@ -218,7 +224,7 @@ public class IcebergTableOperations {
TableIdentifier tableIdentifier = TableIdentifier.of(icebergNS,
tableName);
LoadTableResponse loadTableResponse =
tableOperationDispatcher.updateTable(context, tableIdentifier,
updateTableRequest);
- return IcebergRESTUtils.ok(loadTableResponse);
+ return buildResponseWithETag(loadTableResponse);
});
} catch (Exception e) {
return IcebergExceptionMapper.toRESTResponse(e);
@@ -283,7 +289,8 @@ public class IcebergTableOperations {
@IcebergAuthorizationMetadata(type = RequestType.LOAD_TABLE) @Encoded()
@PathParam("table")
String table,
@DefaultValue("all") @QueryParam("snapshots") String snapshots,
- @HeaderParam(X_ICEBERG_ACCESS_DELEGATION) String accessDelegation) {
+ @HeaderParam(X_ICEBERG_ACCESS_DELEGATION) String accessDelegation,
+ @HeaderParam(IF_NONE_MATCH) String ifNoneMatch) {
String catalogName = IcebergRESTUtils.getCatalogName(prefix);
Namespace icebergNS = RESTUtil.decodeNamespace(namespace);
String tableName = RESTUtil.decodeString(table);
@@ -306,7 +313,12 @@ public class IcebergTableOperations {
new IcebergRequestContext(httpServletRequest(), catalogName,
isCredentialVending);
LoadTableResponse loadTableResponse =
tableOperationDispatcher.loadTable(context, tableIdentifier);
- return IcebergRESTUtils.ok(loadTableResponse);
+ EntityTag etag =
+
generateETag(loadTableResponse.tableMetadata().metadataFileLocation(),
snapshots);
+ if (etag != null && etagMatches(ifNoneMatch, etag)) {
+ return Response.notModified(etag).build();
+ }
+ return buildResponseWithETag(loadTableResponse, etag);
});
} catch (Exception e) {
return IcebergExceptionMapper.toRESTResponse(e);
@@ -508,6 +520,102 @@ public class IcebergTableOperations {
}
}
+ /**
+ * Builds an OK response with the ETag header derived from the table
metadata location.
+ *
+ * @param loadTableResponse the table response to include in the body
+ * @return a Response with ETag header set
+ */
+ private static Response buildResponseWithETag(LoadTableResponse
loadTableResponse) {
+ EntityTag etag =
generateETag(loadTableResponse.tableMetadata().metadataFileLocation());
+ return buildResponseWithETag(loadTableResponse, etag);
+ }
+
+ /**
+ * Builds an OK response with the given ETag header.
+ *
+ * @param loadTableResponse the table response to include in the body
+ * @param etag the pre-computed ETag, may be null
+ * @return a Response with ETag header set if etag is non-null
+ */
+ private static Response buildResponseWithETag(
+ LoadTableResponse loadTableResponse, EntityTag etag) {
+ Response.ResponseBuilder responseBuilder =
+ Response.ok(loadTableResponse, MediaType.APPLICATION_JSON_TYPE);
+ if (etag != null) {
+ responseBuilder.tag(etag);
+ }
+ return responseBuilder.build();
+ }
+
+ /**
+ * Generates an ETag based on the table metadata file location. The ETag is
a SHA-256 hash of the
+ * metadata location, which changes whenever the table metadata is updated.
+ *
+ * @param metadataLocation the metadata file location
+ * @return the generated ETag, or null if generation fails
+ */
+ @VisibleForTesting
+ static EntityTag generateETag(String metadataLocation) {
+ return generateETag(metadataLocation, null);
+ }
+
+ /**
+ * Generates an ETag based on the table metadata file location and snapshot
mode. The ETag is a
+ * SHA-256 hash that incorporates both the metadata location and the
snapshots parameter, ensuring
+ * distinct ETags for different representations of the same table version
(e.g., snapshots=all vs
+ * snapshots=refs).
+ *
+ * @param metadataLocation the metadata file location
+ * @param snapshots the snapshots query parameter value (e.g., "all",
"refs"), may be null
+ * @return the generated ETag, or null if generation fails
+ */
+ @VisibleForTesting
+ static EntityTag generateETag(String metadataLocation, String snapshots) {
+ if (metadataLocation == null) {
+ return null;
+ }
+ try {
+ MessageDigest digest = MessageDigest.getInstance("SHA-256");
+ digest.update(metadataLocation.getBytes(StandardCharsets.UTF_8));
+ if (snapshots != null) {
+ digest.update(snapshots.getBytes(StandardCharsets.UTF_8));
+ }
+ byte[] hash = digest.digest();
+ StringBuilder hexString = new StringBuilder();
+ for (byte b : hash) {
+ String hex = Integer.toHexString(0xff & b);
+ if (hex.length() == 1) {
+ hexString.append('0');
+ }
+ hexString.append(hex);
+ }
+ return new EntityTag(hexString.toString());
+ } catch (NoSuchAlgorithmException e) {
+ LOG.warn("Failed to generate ETag for metadata location: {}",
metadataLocation, e);
+ return null;
+ }
+ }
+
+ /**
+ * Checks if the client's If-None-Match header value matches the current
ETag.
+ *
+ * @param ifNoneMatch the If-None-Match header value from the client
+ * @param etag the current ETag
+ * @return true if the ETag matches (table unchanged), false otherwise
+ */
+ private static boolean etagMatches(String ifNoneMatch, EntityTag etag) {
+ if (ifNoneMatch == null || ifNoneMatch.isEmpty()) {
+ return false;
+ }
+ // Strip quotes if present to compare the raw value
+ String clientEtag = ifNoneMatch.trim();
+ if (clientEtag.startsWith("\"") && clientEtag.endsWith("\"")) {
+ clientEtag = clientEtag.substring(1, clientEtag.length() - 1);
+ }
+ return etag.getValue().equals(clientEtag);
+ }
+
private boolean isCredentialVending(String accessDelegation) {
if (StringUtils.isBlank(accessDelegation)) {
return false;
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergTableOperations.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergTableOperations.java
index 606367ecaa..21063176a2 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergTableOperations.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergTableOperations.java
@@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import java.util.Arrays;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@@ -484,6 +485,13 @@ public class TestIcebergTableOperations extends
IcebergNamespaceTestBase {
return getTableClientBuilder(ns, Optional.of(name)).get();
}
+ private Response doLoadTableWithSnapshots(Namespace ns, String name, String
snapshots) {
+ String path =
+ IcebergRestTestUtil.NAMESPACE_PATH + "/" +
RESTUtil.encodeNamespace(ns) + "/tables/" + name;
+ Map<String, String> queryParams = ImmutableMap.of("snapshots", snapshots);
+ return getIcebergClientBuilder(path, Optional.of(queryParams)).get();
+ }
+
private Response doPlanTableScan(Namespace ns, String tableName,
PlanTableScanRequest request) {
Invocation.Builder builder = getTableClientBuilder(ns,
Optional.of(tableName + "/scan"));
return builder.post(Entity.entity(request,
MediaType.APPLICATION_JSON_TYPE));
@@ -751,4 +759,167 @@ public class TestIcebergTableOperations extends
IcebergNamespaceTestBase {
errorBody.contains("vended-credentials") &&
errorBody.contains("illegal"),
"Error message should mention valid values: " + errorBody);
}
+
+ @ParameterizedTest
+
@MethodSource("org.apache.gravitino.iceberg.service.rest.IcebergRestTestUtil#testNamespaces")
+ void testCreateTableReturnsETag(Namespace namespace) {
+ verifyCreateNamespaceSucc(namespace);
+ Response response = doCreateTable(namespace, "create_etag_foo1");
+ Assertions.assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ String etag = response.getHeaderString("ETag");
+ Assertions.assertNotNull(etag, "ETag header should be present in create
table response");
+ Assertions.assertFalse(etag.isEmpty(), "ETag header should not be empty");
+ }
+
+ @ParameterizedTest
+
@MethodSource("org.apache.gravitino.iceberg.service.rest.IcebergRestTestUtil#testNamespaces")
+ void testUpdateTableReturnsETag(Namespace namespace) {
+ verifyCreateNamespaceSucc(namespace);
+ verifyCreateTableSucc(namespace, "update_etag_foo1");
+ TableMetadata metadata = getTableMeta(namespace, "update_etag_foo1");
+ Response response = doUpdateTable(namespace, "update_etag_foo1", metadata);
+ Assertions.assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ String etag = response.getHeaderString("ETag");
+ Assertions.assertNotNull(etag, "ETag header should be present in update
table response");
+ Assertions.assertFalse(etag.isEmpty(), "ETag header should not be empty");
+ }
+
+ @ParameterizedTest
+
@MethodSource("org.apache.gravitino.iceberg.service.rest.IcebergRestTestUtil#testNamespaces")
+ void testLoadTableReturnsETag(Namespace namespace) {
+ verifyCreateNamespaceSucc(namespace);
+ verifyCreateTableSucc(namespace, "etag_foo1");
+
+ // Load the table and verify ETag header is present
+ Response response = doLoadTable(namespace, "etag_foo1");
+ Assertions.assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ String etag = response.getHeaderString("ETag");
+ Assertions.assertNotNull(etag, "ETag header should be present in load
table response");
+ Assertions.assertFalse(etag.isEmpty(), "ETag header should not be empty");
+ }
+
+ @ParameterizedTest
+
@MethodSource("org.apache.gravitino.iceberg.service.rest.IcebergRestTestUtil#testNamespaces")
+ void testLoadTableReturns304WhenETagMatches(Namespace namespace) {
+ verifyCreateNamespaceSucc(namespace);
+ verifyCreateTableSucc(namespace, "etag_304_foo1");
+
+ // First, load the table to get the ETag
+ Response firstResponse = doLoadTable(namespace, "etag_304_foo1");
+ Assertions.assertEquals(Status.OK.getStatusCode(),
firstResponse.getStatus());
+ String etag = firstResponse.getHeaderString("ETag");
+ Assertions.assertNotNull(etag, "ETag header should be present");
+
+ // Second, load the table with If-None-Match header set to the ETag
+ Response secondResponse =
+ getTableClientBuilder(namespace, Optional.of("etag_304_foo1"))
+ .header(IcebergTableOperations.IF_NONE_MATCH, etag)
+ .get();
+ Assertions.assertEquals(
+ Status.NOT_MODIFIED.getStatusCode(),
+ secondResponse.getStatus(),
+ "Should return 304 when ETag matches");
+ }
+
+ @ParameterizedTest
+
@MethodSource("org.apache.gravitino.iceberg.service.rest.IcebergRestTestUtil#testNamespaces")
+ void testLoadTableReturns200WhenETagDoesNotMatch(Namespace namespace) {
+ verifyCreateNamespaceSucc(namespace);
+ verifyCreateTableSucc(namespace, "etag_mismatch_foo1");
+
+ // Load with a non-matching If-None-Match header
+ Response response =
+ getTableClientBuilder(namespace, Optional.of("etag_mismatch_foo1"))
+ .header(IcebergTableOperations.IF_NONE_MATCH,
"\"non-matching-etag-value\"")
+ .get();
+ Assertions.assertEquals(
+ Status.OK.getStatusCode(),
+ response.getStatus(),
+ "Should return 200 when ETag does not match");
+ String etag = response.getHeaderString("ETag");
+ Assertions.assertNotNull(etag, "ETag header should be present");
+ }
+
+ @ParameterizedTest
+
@MethodSource("org.apache.gravitino.iceberg.service.rest.IcebergRestTestUtil#testNamespaces")
+ void testLoadTableETagChangesAfterUpdate(Namespace namespace) {
+ verifyCreateNamespaceSucc(namespace);
+ verifyCreateTableSucc(namespace, "etag_update_foo1");
+
+ // Load the table and get the initial ETag
+ Response firstResponse = doLoadTable(namespace, "etag_update_foo1");
+ Assertions.assertEquals(Status.OK.getStatusCode(),
firstResponse.getStatus());
+ String firstEtag = firstResponse.getHeaderString("ETag");
+ Assertions.assertNotNull(firstEtag, "ETag header should be present");
+
+ // Update the table
+ TableMetadata metadata = getTableMeta(namespace, "etag_update_foo1");
+ verifyUpdateSucc(namespace, "etag_update_foo1", metadata);
+
+ // Load the table again and verify the ETag has changed
+ Response secondResponse = doLoadTable(namespace, "etag_update_foo1");
+ Assertions.assertEquals(Status.OK.getStatusCode(),
secondResponse.getStatus());
+ String secondEtag = secondResponse.getHeaderString("ETag");
+ Assertions.assertNotNull(secondEtag, "ETag header should be present after
update");
+ Assertions.assertNotEquals(
+ firstEtag, secondEtag, "ETag should change after table metadata is
updated");
+
+ // Verify old ETag no longer returns 304
+ Response thirdResponse =
+ getTableClientBuilder(namespace, Optional.of("etag_update_foo1"))
+ .header(IcebergTableOperations.IF_NONE_MATCH, firstEtag)
+ .get();
+ Assertions.assertEquals(
+ Status.OK.getStatusCode(),
+ thirdResponse.getStatus(),
+ "Old ETag should not return 304 after table update");
+ }
+
+ @ParameterizedTest
+
@MethodSource("org.apache.gravitino.iceberg.service.rest.IcebergRestTestUtil#testNamespaces")
+ void testLoadTableETagConsistency(Namespace namespace) {
+ verifyCreateNamespaceSucc(namespace);
+ verifyCreateTableSucc(namespace, "etag_consistent_foo1");
+
+ // Load the table twice and verify the ETag is the same
+ Response firstResponse = doLoadTable(namespace, "etag_consistent_foo1");
+ String firstEtag = firstResponse.getHeaderString("ETag");
+
+ Response secondResponse = doLoadTable(namespace, "etag_consistent_foo1");
+ String secondEtag = secondResponse.getHeaderString("ETag");
+
+ Assertions.assertEquals(
+ firstEtag, secondEtag, "ETag should be consistent for the same table
metadata");
+ }
+
+ @ParameterizedTest
+
@MethodSource("org.apache.gravitino.iceberg.service.rest.IcebergRestTestUtil#testNamespaces")
+ void testLoadTableETagDiffersForSnapshotsParam(Namespace namespace) {
+ verifyCreateNamespaceSucc(namespace);
+ verifyCreateTableSucc(namespace, "etag_snapshots_foo1");
+
+ // Load with snapshots=all (default)
+ Response allResponse = doLoadTableWithSnapshots(namespace,
"etag_snapshots_foo1", "all");
+ Assertions.assertEquals(Status.OK.getStatusCode(),
allResponse.getStatus());
+ String allEtag = allResponse.getHeaderString("ETag");
+ Assertions.assertNotNull(allEtag, "ETag header should be present for
snapshots=all");
+
+ // Load with snapshots=refs
+ Response refsResponse = doLoadTableWithSnapshots(namespace,
"etag_snapshots_foo1", "refs");
+ Assertions.assertEquals(Status.OK.getStatusCode(),
refsResponse.getStatus());
+ String refsEtag = refsResponse.getHeaderString("ETag");
+ Assertions.assertNotNull(refsEtag, "ETag header should be present for
snapshots=refs");
+
+ // ETags should differ for different snapshots values
+ Assertions.assertNotEquals(
+ allEtag,
+ refsEtag,
+ "ETags should be distinct for different snapshots query parameter
values");
+
+ // Loading with the same snapshots value should produce the same ETag
+ Response allResponse2 = doLoadTableWithSnapshots(namespace,
"etag_snapshots_foo1", "all");
+ String allEtag2 = allResponse2.getHeaderString("ETag");
+ Assertions.assertEquals(
+ allEtag, allEtag2, "ETag should be consistent for the same snapshots
value");
+ }
}
