This is an automated email from the ASF dual-hosted git repository.
emaynard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new 8b5dfa97a Added freshness aware table loading using metadata file
location for ETag (#1037)
8b5dfa97a is described below
commit 8b5dfa97a8e50a53412d2ded40f2e4db5a40cc42
Author: Mansehaj Singh <[email protected]>
AuthorDate: Tue Apr 1 16:11:57 2025 -0700
Added freshness aware table loading using metadata file location for ETag
(#1037)
* Pulled in iceberg 1.8.0 spec changes for freshness aware table loading
and added feature to Polaris
* Changed etag support to use entityId:version tuple
* fixed getresponse call
* Changed etagged response to record and gave default implementation to
ETaggableEntity
* Made iceberg rest spec docs clearer
* Added HTTP Compliant ETag and IfNoneMatch representations and separated
persistence from etag logic
* Changed ETag to be a record and improved semantics of IfNoneMatch
* Fixed semantics of if none match
* Removed ETag representation, consolidated in IfNoneMatch
* fixed if none match parsing
* Added table entity retrieval method to table operations
* removed accidental commit of pycache folders
* Fixed formatting
* Changed to use metadata location hash
* Ran formatting
* use sha256
* Moved out ETag functions to utility class and removed
ETaggedLoadTableResponse
* Addressed comments
* Fixed IcebergTableLikeEntity package rename
---
.../it/test/PolarisRestCatalogIntegrationTest.java | 151 +++++++++++++++++++++
.../IcebergCatalogHandlerWrapperAuthzTest.java | 97 +++++++++++++
.../catalog/iceberg/IcebergCatalogAdapter.java | 58 +++++++-
.../iceberg/IcebergCatalogHandlerWrapper.java | 92 ++++++++++++-
.../polaris/service/http/IcebergHttpUtil.java | 42 ++++++
.../apache/polaris/service/http/IfNoneMatch.java | 113 +++++++++++++++
.../polaris/service/http/IfNoneMatchTest.java | 132 ++++++++++++++++++
7 files changed, 675 insertions(+), 10 deletions(-)
diff --git
a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java
b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java
index 83c9791b3..f7466b77f 100644
---
a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java
+++
b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java
@@ -25,6 +25,7 @@ import static
org.assertj.core.api.Assertions.assertThatThrownBy;
import com.google.common.collect.ImmutableMap;
import jakarta.ws.rs.client.Entity;
import jakarta.ws.rs.client.Invocation;
+import jakarta.ws.rs.core.HttpHeaders;
import jakarta.ws.rs.core.Response;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
@@ -56,6 +57,7 @@ import org.apache.iceberg.exceptions.ForbiddenException;
import org.apache.iceberg.expressions.Expressions;
import org.apache.iceberg.io.ResolvingFileIO;
import org.apache.iceberg.rest.RESTCatalog;
+import org.apache.iceberg.rest.requests.CreateTableRequest;
import org.apache.iceberg.rest.responses.ErrorResponse;
import org.apache.iceberg.types.Types;
import org.apache.polaris.core.admin.model.AwsStorageConfigInfo;
@@ -92,6 +94,7 @@ import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -646,6 +649,154 @@ public class PolarisRestCatalogIntegrationTest extends
CatalogTests<RESTCatalog>
}
}
+ /**
+ * Register a table. Then, invoke an initial loadTable request to fetch and
ensure ETag is
+ * present. Then, invoke a second loadTable to ensure that ETag is matched.
+ */
+ @Test
+ @Disabled("Enable once ETag support is available in the API for loadTable.")
+ public void testLoadTableTwiceWithETag() {
+ // TODO: Re-enable test once spec is up to date with ETag change for
loadTable in Iceberg
+
+ Namespace ns1 = Namespace.of("ns1");
+ restCatalog.createNamespace(ns1);
+ TableMetadata tableMetadata =
+ TableMetadata.newTableMetadata(
+ new Schema(List.of(Types.NestedField.of(1, false, "col1", new
Types.StringType()))),
+ PartitionSpec.unpartitioned(),
+ "file:///tmp/ns1/my_table",
+ Map.of());
+ try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) {
+ resolvingFileIO.initialize(Map.of());
+ resolvingFileIO.setConf(new Configuration());
+ String fileLocation =
"file:///tmp/ns1/my_table/metadata/v1.metadata.json";
+ TableMetadataParser.write(tableMetadata,
resolvingFileIO.newOutputFile(fileLocation));
+ restCatalog.registerTable(TableIdentifier.of(ns1, "my_table_etagged"),
fileLocation);
+ Invocation invocation =
+ catalogApi
+ .request("v1/" + currentCatalogName +
"/namespaces/ns1/tables/my_table_etagged")
+ .build("GET");
+ try (Response initialLoadTable = invocation.invoke()) {
+
assertThat(initialLoadTable.getHeaders()).containsKey(HttpHeaders.ETAG);
+ String etag =
initialLoadTable.getHeaders().getFirst(HttpHeaders.ETAG).toString();
+
+ Invocation etaggedInvocation =
+ catalogApi
+ .request("v1/" + currentCatalogName +
"/namespaces/ns1/tables/my_table_etagged")
+ .header(HttpHeaders.IF_NONE_MATCH, etag)
+ .build("GET");
+
+ try (Response etaggedLoadTable = etaggedInvocation.invoke()) {
+ assertThat(etaggedLoadTable.getStatus())
+ .isEqualTo(Response.Status.NOT_MODIFIED.getStatusCode());
+ }
+ } finally {
+ resolvingFileIO.deleteFile(fileLocation);
+ }
+ }
+ }
+
+ /**
+ * Invoke an initial registerTable request to fetch and ensure ETag is
present. Then, invoke a
+ * second loadTable to ensure that ETag is matched.
+ */
+ @Test
+ @Disabled("Enable once ETag support is available in the API for loadTable.")
+ public void testRegisterAndLoadTableWithReturnedETag() {
+ // TODO: Re-enable test once spec is up to date with ETag change for
loadTable in Iceberg
+
+ Namespace ns1 = Namespace.of("ns1");
+ restCatalog.createNamespace(ns1);
+ TableMetadata tableMetadata =
+ TableMetadata.newTableMetadata(
+ new Schema(List.of(Types.NestedField.of(1, false, "col1", new
Types.StringType()))),
+ PartitionSpec.unpartitioned(),
+ "file:///tmp/ns1/my_table",
+ Map.of());
+ try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) {
+ resolvingFileIO.initialize(Map.of());
+ resolvingFileIO.setConf(new Configuration());
+ String fileLocation =
"file:///tmp/ns1/my_table/metadata/v1.metadata.json";
+ TableMetadataParser.write(tableMetadata,
resolvingFileIO.newOutputFile(fileLocation));
+
+ Invocation registerInvocation =
+ catalogApi
+ .request("v1/" + currentCatalogName + "/namespaces/ns1/register")
+ .buildPost(
+ Entity.json(
+ Map.of("name", "my_etagged_table", "metadata-location",
fileLocation)));
+ try (Response registerResponse = registerInvocation.invoke()) {
+
assertThat(registerResponse.getHeaders()).containsKey(HttpHeaders.ETAG);
+ String etag =
registerResponse.getHeaders().getFirst(HttpHeaders.ETAG).toString();
+
+ Invocation etaggedInvocation =
+ catalogApi
+ .request("v1/" + currentCatalogName +
"/namespaces/ns1/tables/my_etagged_table")
+ .header(HttpHeaders.IF_NONE_MATCH, etag)
+ .build("GET");
+
+ try (Response etaggedLoadTable = etaggedInvocation.invoke()) {
+ assertThat(etaggedLoadTable.getStatus())
+ .isEqualTo(Response.Status.NOT_MODIFIED.getStatusCode());
+ }
+
+ } finally {
+ resolvingFileIO.deleteFile(fileLocation);
+ }
+ }
+ }
+
+ @Test
+ @Disabled("Enable once ETag support is available in the API for loadTable.")
+ public void testCreateAndLoadTableWithReturnedEtag() {
+ // TODO: Re-enable test once spec is up to date with ETag change for
loadTable in Iceberg
+
+ Namespace ns1 = Namespace.of("ns1");
+ restCatalog.createNamespace(ns1);
+ TableMetadata tableMetadata =
+ TableMetadata.newTableMetadata(
+ new Schema(List.of(Types.NestedField.of(1, false, "col1", new
Types.StringType()))),
+ PartitionSpec.unpartitioned(),
+ "file:///tmp/ns1/my_table",
+ Map.of());
+ try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) {
+ resolvingFileIO.initialize(Map.of());
+ resolvingFileIO.setConf(new Configuration());
+ String fileLocation =
"file:///tmp/ns1/my_table/metadata/v1.metadata.json";
+ TableMetadataParser.write(tableMetadata,
resolvingFileIO.newOutputFile(fileLocation));
+
+ Invocation createInvocation =
+ catalogApi
+ .request("v1/" + currentCatalogName + "/namespaces/ns1/tables")
+ .buildPost(
+ Entity.json(
+ CreateTableRequest.builder()
+ .withName("my_etagged_table")
+ .withLocation(tableMetadata.location())
+ .withPartitionSpec(tableMetadata.spec())
+ .withSchema(tableMetadata.schema())
+ .withWriteOrder(tableMetadata.sortOrder())
+ .build()));
+ try (Response createResponse = createInvocation.invoke()) {
+ assertThat(createResponse.getHeaders()).containsKey(HttpHeaders.ETAG);
+ String etag =
createResponse.getHeaders().getFirst(HttpHeaders.ETAG).toString();
+
+ Invocation etaggedInvocation =
+ catalogApi
+ .request("v1/" + currentCatalogName +
"/namespaces/ns1/tables/my_etagged_table")
+ .header(HttpHeaders.IF_NONE_MATCH, etag)
+ .build("GET");
+
+ try (Response etaggedLoadTable = etaggedInvocation.invoke()) {
+ assertThat(etaggedLoadTable.getStatus())
+ .isEqualTo(Response.Status.NOT_MODIFIED.getStatusCode());
+ }
+ } finally {
+ resolvingFileIO.deleteFile(fileLocation);
+ }
+ }
+ }
+
@Test
public void testSendNotificationInternalCatalog() {
Map<String, String> payload =
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerWrapperAuthzTest.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerWrapperAuthzTest.java
index e028e5e3b..1e2d05bd6 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerWrapperAuthzTest.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerWrapperAuthzTest.java
@@ -69,6 +69,7 @@ import
org.apache.polaris.service.catalog.io.DefaultFileIOFactory;
import org.apache.polaris.service.config.RealmEntityManagerFactory;
import org.apache.polaris.service.context.CallContextCatalogFactory;
import org.apache.polaris.service.context.PolarisCallContextCatalogFactory;
+import org.apache.polaris.service.http.IfNoneMatch;
import org.apache.polaris.service.quarkus.admin.PolarisAuthzTestBase;
import org.apache.polaris.service.types.NotificationRequest;
import org.apache.polaris.service.types.NotificationType;
@@ -865,6 +866,35 @@ public class IcebergCatalogHandlerWrapperAuthzTest extends
PolarisAuthzTestBase
() -> newWrapper().loadTable(TABLE_NS1A_2, "all"));
}
+ @Test
+ public void testLoadTableIfStaleSufficientPrivileges() {
+ doTestSufficientPrivileges(
+ List.of(
+ PolarisPrivilege.TABLE_READ_PROPERTIES,
+ PolarisPrivilege.TABLE_WRITE_PROPERTIES,
+ PolarisPrivilege.TABLE_READ_DATA,
+ PolarisPrivilege.TABLE_WRITE_DATA,
+ PolarisPrivilege.TABLE_FULL_METADATA,
+ PolarisPrivilege.CATALOG_MANAGE_CONTENT),
+ () ->
+ newWrapper().loadTableIfStale(TABLE_NS1A_2,
IfNoneMatch.fromHeader("W/\"0:0\""), "all"),
+ null /* cleanupAction */);
+ }
+
+ @Test
+ public void testLoadTableIfStaleInsufficientPermissions() {
+ doTestInsufficientPrivileges(
+ List.of(
+ PolarisPrivilege.NAMESPACE_FULL_METADATA,
+ PolarisPrivilege.VIEW_FULL_METADATA,
+ PolarisPrivilege.TABLE_CREATE,
+ PolarisPrivilege.TABLE_LIST,
+ PolarisPrivilege.TABLE_DROP),
+ () ->
+ newWrapper()
+ .loadTableIfStale(TABLE_NS1A_2,
IfNoneMatch.fromHeader("W/\"0:0\""), "all"));
+ }
+
@Test
public void testLoadTableWithReadAccessDelegationSufficientPrivileges() {
doTestSufficientPrivileges(
@@ -920,6 +950,73 @@ public class IcebergCatalogHandlerWrapperAuthzTest extends
PolarisAuthzTestBase
() -> newWrapper().loadTableWithAccessDelegation(TABLE_NS1A_2, "all"));
}
+ @Test
+ public void
testLoadTableWithReadAccessDelegationIfStaleSufficientPrivileges() {
+ doTestSufficientPrivileges(
+ List.of(
+ PolarisPrivilege.TABLE_READ_DATA,
+ PolarisPrivilege.TABLE_WRITE_DATA,
+ PolarisPrivilege.CATALOG_MANAGE_CONTENT),
+ () ->
+ newWrapper()
+ .loadTableWithAccessDelegationIfStale(
+ TABLE_NS1A_2, IfNoneMatch.fromHeader("W/\"0:0\""), "all"),
+ null /* cleanupAction */);
+ }
+
+ @Test
+ public void
testLoadTableWithReadAccessDelegationIfStaleInsufficientPermissions() {
+ doTestInsufficientPrivileges(
+ List.of(
+ PolarisPrivilege.NAMESPACE_FULL_METADATA,
+ PolarisPrivilege.VIEW_FULL_METADATA,
+ PolarisPrivilege.TABLE_FULL_METADATA,
+ PolarisPrivilege.TABLE_READ_PROPERTIES,
+ PolarisPrivilege.TABLE_WRITE_PROPERTIES,
+ PolarisPrivilege.TABLE_CREATE,
+ PolarisPrivilege.TABLE_LIST,
+ PolarisPrivilege.TABLE_DROP),
+ () ->
+ newWrapper()
+ .loadTableWithAccessDelegationIfStale(
+ TABLE_NS1A_2, IfNoneMatch.fromHeader("W/\"0:0\""), "all"));
+ }
+
+ @Test
+ public void
testLoadTableWithWriteAccessDelegationIfStaleSufficientPrivileges() {
+ doTestSufficientPrivileges(
+ List.of(
+ // TODO: Once we give different creds for read/write privilege,
move this
+ // TABLE_READ_DATA into a special-case test; with only
TABLE_READ_DATA we'd expet
+ // to receive a read-only credential.
+ PolarisPrivilege.TABLE_READ_DATA,
+ PolarisPrivilege.TABLE_WRITE_DATA,
+ PolarisPrivilege.CATALOG_MANAGE_CONTENT),
+ () ->
+ newWrapper()
+ .loadTableWithAccessDelegationIfStale(
+ TABLE_NS1A_2, IfNoneMatch.fromHeader("W/\"0:0\""), "all"),
+ null /* cleanupAction */);
+ }
+
+ @Test
+ public void
testLoadTableWithWriteAccessDelegationIfStaleInsufficientPermissions() {
+ doTestInsufficientPrivileges(
+ List.of(
+ PolarisPrivilege.NAMESPACE_FULL_METADATA,
+ PolarisPrivilege.VIEW_FULL_METADATA,
+ PolarisPrivilege.TABLE_FULL_METADATA,
+ PolarisPrivilege.TABLE_READ_PROPERTIES,
+ PolarisPrivilege.TABLE_WRITE_PROPERTIES,
+ PolarisPrivilege.TABLE_CREATE,
+ PolarisPrivilege.TABLE_LIST,
+ PolarisPrivilege.TABLE_DROP),
+ () ->
+ newWrapper()
+ .loadTableWithAccessDelegationIfStale(
+ TABLE_NS1A_2, IfNoneMatch.fromHeader("W/\"0:0\""), "all"));
+ }
+
@Test
public void testUpdateTableSufficientPrivileges() {
doTestSufficientPrivileges(
diff --git
a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java
b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java
index 7cb4f2f95..4da568322 100644
---
a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java
+++
b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java
@@ -26,6 +26,8 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import jakarta.enterprise.context.RequestScoped;
import jakarta.inject.Inject;
+import jakarta.ws.rs.WebApplicationException;
+import jakarta.ws.rs.core.HttpHeaders;
import jakarta.ws.rs.core.Response;
import jakarta.ws.rs.core.SecurityContext;
import java.net.URLEncoder;
@@ -70,6 +72,8 @@ import org.apache.polaris.service.catalog.CatalogPrefixParser;
import org.apache.polaris.service.catalog.api.IcebergRestCatalogApiService;
import
org.apache.polaris.service.catalog.api.IcebergRestConfigurationApiService;
import org.apache.polaris.service.context.CallContextCatalogFactory;
+import org.apache.polaris.service.http.IcebergHttpUtil;
+import org.apache.polaris.service.http.IfNoneMatch;
import org.apache.polaris.service.types.CommitTableRequest;
import org.apache.polaris.service.types.CommitViewRequest;
import org.apache.polaris.service.types.NotificationRequest;
@@ -307,9 +311,21 @@ public class IcebergCatalogAdapter
.build();
}
} else if (delegationModes.isEmpty()) {
- return Response.ok(catalog.createTableDirect(ns,
createTableRequest)).build();
+ LoadTableResponse response = catalog.createTableDirect(ns,
createTableRequest);
+ return Response.ok(response)
+ .header(
+ HttpHeaders.ETAG,
+ IcebergHttpUtil.generateETagForMetadataFileLocation(
+ response.metadataLocation()))
+ .build();
} else {
- return
Response.ok(catalog.createTableDirectWithWriteDelegation(ns,
createTableRequest))
+ LoadTableResponse response =
+ catalog.createTableDirectWithWriteDelegation(ns,
createTableRequest);
+ return Response.ok(response)
+ .header(
+ HttpHeaders.ETAG,
+ IcebergHttpUtil.generateETagForMetadataFileLocation(
+ response.metadataLocation()))
.build();
}
});
@@ -341,16 +357,38 @@ public class IcebergCatalogAdapter
parseAccessDelegationModes(accessDelegationMode);
Namespace ns = decodeNamespace(namespace);
TableIdentifier tableIdentifier = TableIdentifier.of(ns,
RESTUtil.decodeString(table));
+
+ // TODO: Populate with header value from parameter once the generated
interface
+ // contains the if-none-match header
+ IfNoneMatch ifNoneMatch = IfNoneMatch.fromHeader(null);
+
+ if (ifNoneMatch.isWildcard()) {
+ throw new BadRequestException("If-None-Match may not take the value of
'*'");
+ }
+
return withCatalog(
securityContext,
prefix,
catalog -> {
+ LoadTableResponse response;
+
if (delegationModes.isEmpty()) {
- return Response.ok(catalog.loadTable(tableIdentifier,
snapshots)).build();
+ response =
+ catalog
+ .loadTableIfStale(tableIdentifier, ifNoneMatch, snapshots)
+ .orElseThrow(() -> new
WebApplicationException(Response.Status.NOT_MODIFIED));
} else {
- return
Response.ok(catalog.loadTableWithAccessDelegation(tableIdentifier, snapshots))
- .build();
+ response =
+ catalog
+ .loadTableWithAccessDelegationIfStale(tableIdentifier,
ifNoneMatch, snapshots)
+ .orElseThrow(() -> new
WebApplicationException(Response.Status.NOT_MODIFIED));
}
+
+ return Response.ok(response)
+ .header(
+ HttpHeaders.ETAG,
+
IcebergHttpUtil.generateETagForMetadataFileLocation(response.metadataLocation()))
+ .build();
});
}
@@ -406,7 +444,15 @@ public class IcebergCatalogAdapter
return withCatalog(
securityContext,
prefix,
- catalog -> Response.ok(catalog.registerTable(ns,
registerTableRequest)).build());
+ catalog -> {
+ LoadTableResponse response = catalog.registerTable(ns,
registerTableRequest);
+
+ return Response.ok(response)
+ .header(
+ HttpHeaders.ETAG,
+
IcebergHttpUtil.generateETagForMetadataFileLocation(response.metadataLocation()))
+ .build();
+ });
}
@Override
diff --git
a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerWrapper.java
b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerWrapper.java
index fab2453e7..284001fa5 100644
---
a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerWrapper.java
+++
b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerWrapper.java
@@ -80,6 +80,7 @@ import org.apache.polaris.core.context.CallContext;
import org.apache.polaris.core.entity.CatalogEntity;
import org.apache.polaris.core.entity.PolarisEntitySubType;
import org.apache.polaris.core.entity.PolarisEntityType;
+import org.apache.polaris.core.entity.table.IcebergTableLikeEntity;
import org.apache.polaris.core.persistence.PolarisEntityManager;
import org.apache.polaris.core.persistence.PolarisMetaStoreManager;
import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper;
@@ -92,6 +93,8 @@ import
org.apache.polaris.core.persistence.resolver.ResolverStatus;
import org.apache.polaris.core.storage.PolarisStorageActions;
import org.apache.polaris.service.catalog.SupportsNotifications;
import org.apache.polaris.service.context.CallContextCatalogFactory;
+import org.apache.polaris.service.http.IcebergHttpUtil;
+import org.apache.polaris.service.http.IfNoneMatch;
import org.apache.polaris.service.types.NotificationRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -545,10 +548,17 @@ public class IcebergCatalogHandlerWrapper implements
AutoCloseable {
return CatalogHandlers.listTables(baseCatalog, namespace);
}
+ /**
+ * Create a table.
+ *
+ * @param namespace the namespace to create the table in
+ * @param request the table creation request
+ * @return ETagged {@link LoadTableResponse} to uniquely identify the table
metadata
+ */
public LoadTableResponse createTableDirect(Namespace namespace,
CreateTableRequest request) {
PolarisAuthorizableOperation op =
PolarisAuthorizableOperation.CREATE_TABLE_DIRECT;
- authorizeCreateTableLikeUnderNamespaceOperationOrThrow(
- op, TableIdentifier.of(namespace, request.name()));
+ TableIdentifier identifier = TableIdentifier.of(namespace, request.name());
+ authorizeCreateTableLikeUnderNamespaceOperationOrThrow(op, identifier);
CatalogEntity catalog =
CatalogEntity.of(
@@ -562,6 +572,13 @@ public class IcebergCatalogHandlerWrapper implements
AutoCloseable {
return CatalogHandlers.createTable(baseCatalog, namespace, request);
}
+ /**
+ * Create a table.
+ *
+ * @param namespace the namespace to create the table in
+ * @param request the table creation request
+ * @return ETagged {@link LoadTableResponse} to uniquely identify the table
metadata
+ */
public LoadTableResponse createTableDirectWithWriteDelegation(
Namespace namespace, CreateTableRequest request) {
PolarisAuthorizableOperation op =
@@ -722,6 +739,13 @@ public class IcebergCatalogHandlerWrapper implements
AutoCloseable {
return responseBuilder.build();
}
+ /**
+ * Register a table.
+ *
+ * @param namespace The namespace to register the table in
+ * @param request the register table request
+ * @return ETagged {@link LoadTableResponse} to uniquely identify the table
metadata
+ */
public LoadTableResponse registerTable(Namespace namespace,
RegisterTableRequest request) {
PolarisAuthorizableOperation op =
PolarisAuthorizableOperation.REGISTER_TABLE;
authorizeCreateTableLikeUnderNamespaceOperationOrThrow(
@@ -768,16 +792,67 @@ public class IcebergCatalogHandlerWrapper implements
AutoCloseable {
&& notificationCatalog.sendNotification(identifier, request);
}
+ /**
+ * Fetch the metastore table entity for the given table identifier
+ *
+ * @param tableIdentifier The identifier of the table
+ * @return the Polaris table entity for the table
+ */
+ private IcebergTableLikeEntity getTableEntity(TableIdentifier
tableIdentifier) {
+ PolarisResolvedPathWrapper target =
+ resolutionManifest.getPassthroughResolvedPath(tableIdentifier);
+
+ return IcebergTableLikeEntity.of(target.getRawLeafEntity());
+ }
+
public LoadTableResponse loadTable(TableIdentifier tableIdentifier, String
snapshots) {
+ return loadTableIfStale(tableIdentifier, null, snapshots).get();
+ }
+
+ /**
+ * Attempt to perform a loadTable operation only when the specified set of
eTags do not match the
+ * current state of the table metadata.
+ *
+ * @param tableIdentifier The identifier of the table to load
+ * @param ifNoneMatch set of entity-tags to check the metadata against for
staleness
+ * @param snapshots
+ * @return {@link Optional#empty()} if the ETag is current, an {@link
Optional} containing the
+ * load table response, otherwise
+ */
+ public Optional<LoadTableResponse> loadTableIfStale(
+ TableIdentifier tableIdentifier, IfNoneMatch ifNoneMatch, String
snapshots) {
PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LOAD_TABLE;
authorizeBasicTableLikeOperationOrThrow(
op, PolarisEntitySubType.ICEBERG_TABLE, tableIdentifier);
- return CatalogHandlers.loadTable(baseCatalog, tableIdentifier);
+ IcebergTableLikeEntity tableEntity = getTableEntity(tableIdentifier);
+ String tableEntityTag =
+
IcebergHttpUtil.generateETagForMetadataFileLocation(tableEntity.getMetadataLocation());
+
+ if (ifNoneMatch != null && ifNoneMatch.anyMatch(tableEntityTag)) {
+ return Optional.empty();
+ }
+
+ return Optional.of(CatalogHandlers.loadTable(baseCatalog,
tableIdentifier));
}
public LoadTableResponse loadTableWithAccessDelegation(
TableIdentifier tableIdentifier, String snapshots) {
+ return loadTableWithAccessDelegationIfStale(tableIdentifier, null,
snapshots).get();
+ }
+
+ /**
+ * Attempt to perform a loadTable operation with access delegation only when
the if none of the
+ * provided eTags match the current state of the table metadata.
+ *
+ * @param tableIdentifier The identifier of the table to load
+ * @param ifNoneMatch set of entity-tags to check the metadata against for
staleness
+ * @param snapshots
+ * @return {@link Optional#empty()} if the ETag is current, an {@link
Optional} containing the
+ * load table response, otherwise
+ */
+ public Optional<LoadTableResponse> loadTableWithAccessDelegationIfStale(
+ TableIdentifier tableIdentifier, IfNoneMatch ifNoneMatch, String
snapshots) {
// Here we have a single method that falls through multiple candidate
// PolarisAuthorizableOperations because instead of identifying the
desired operation up-front
// and
@@ -822,6 +897,14 @@ public class IcebergCatalogHandlerWrapper implements
AutoCloseable {
FeatureConfiguration.ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING.catalogConfig());
}
+ IcebergTableLikeEntity tableEntity = getTableEntity(tableIdentifier);
+ String tableETag =
+
IcebergHttpUtil.generateETagForMetadataFileLocation(tableEntity.getMetadataLocation());
+
+ if (ifNoneMatch != null && ifNoneMatch.anyMatch(tableETag)) {
+ return Optional.empty();
+ }
+
// TODO: Find a way for the configuration or caller to better express
whether to fail or omit
// when data-access is specified but access delegation grants are not
found.
Table table = baseCatalog.loadTable(tableIdentifier);
@@ -840,7 +923,8 @@ public class IcebergCatalogHandlerWrapper implements
AutoCloseable {
credentialDelegation.getCredentialConfig(
tableIdentifier, tableMetadata, actionsRequested));
}
- return responseBuilder.build();
+
+ return Optional.of(responseBuilder.build());
} else if (table instanceof BaseMetadataTable) {
// metadata tables are loaded on the client side, return
NoSuchTableException for now
throw new NoSuchTableException("Table does not exist: %s",
tableIdentifier.toString());
diff --git
a/service/common/src/main/java/org/apache/polaris/service/http/IcebergHttpUtil.java
b/service/common/src/main/java/org/apache/polaris/service/http/IcebergHttpUtil.java
new file mode 100644
index 000000000..88c056aa7
--- /dev/null
+++
b/service/common/src/main/java/org/apache/polaris/service/http/IcebergHttpUtil.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.http;
+
+import org.apache.commons.codec.digest.DigestUtils;
+
+/** Utility class that encapsulates logic pertaining to Iceberg REST specific
concepts. */
+public class IcebergHttpUtil {
+
+ private IcebergHttpUtil() {}
+
+ /**
+ * Generate an ETag from an Iceberg metadata file location
+ *
+ * @param metadataFileLocation
+ * @return the generated ETag
+ */
+ public static String generateETagForMetadataFileLocation(String
metadataFileLocation) {
+ // Use hash of metadata location since we don't want clients to use the
ETag to try to extract
+ // the metadata file location
+ String hashedMetadataFileLocation =
DigestUtils.sha256Hex(metadataFileLocation);
+
+ // always issue a weak ETag since we semantically compare metadata, not
its content byte-by-byte
+ return "W/\"" + hashedMetadataFileLocation + "\"";
+ }
+}
diff --git
a/service/common/src/main/java/org/apache/polaris/service/http/IfNoneMatch.java
b/service/common/src/main/java/org/apache/polaris/service/http/IfNoneMatch.java
new file mode 100644
index 000000000..c8fa10434
--- /dev/null
+++
b/service/common/src/main/java/org/apache/polaris/service/http/IfNoneMatch.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.http;
+
+import jakarta.annotation.Nonnull;
+import java.util.List;
+import java.util.regex.MatchResult;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/** Logical representation of an HTTP compliant If-None-Match header. */
+public record IfNoneMatch(boolean isWildcard, @Nonnull List<String> eTags) {
+
+ private static final String WILDCARD_HEADER_VALUE = "*";
+
+ // Matches but does not capture any content of an ETag
+ private static final String ETAG_REGEX = "(?:W/)?\"(?:[^\"]*)\"";
+
+ // Builds comma separated list of ETags and disallows trailing comma
+ private static final String IF_NONE_MATCH_REGEX =
+ String.format("(?:%s, )*%s", ETAG_REGEX, ETAG_REGEX);
+
+ // Wraps pattern in capture group to capture entire ETag
+ private static final Pattern ETAG_PATTERN =
Pattern.compile(String.format("(%s)", ETAG_REGEX));
+
+ // Used to validate entire header pattern
+ private static final Pattern IF_NONE_MATCH_PATTERN =
Pattern.compile(IF_NONE_MATCH_REGEX);
+
+ public static final IfNoneMatch EMPTY = new IfNoneMatch(List.of());
+
+ public static final IfNoneMatch WILDCARD = new IfNoneMatch(true, List.of());
+
+ public IfNoneMatch(List<String> etags) {
+ this(false, etags);
+ }
+
+ public IfNoneMatch {
+ if (isWildcard && !eTags.isEmpty()) {
+ // if the header is a wildcard, it must not be constructed with
+ // any etags, if it is not a wildcard, it can still contain no ETags, so
+ // the converse is not true
+ throw new IllegalArgumentException(
+ "Invalid representation for If-None-Match header. If-None-Match
cannot contain ETags if it takes "
+ + "the wildcard value '*'");
+ }
+
+ for (String etag : eTags) {
+ Matcher matcher = ETAG_PATTERN.matcher(etag);
+
+ if (!matcher.matches()) {
+ throw new IllegalArgumentException("Invalid ETag representation: " +
etag);
+ }
+ }
+ }
+
+ /**
+ * Parses the raw content of an If-None-Match header into the logical
representation
+ *
+ * @param rawValue The raw value of the If-None-Match header
+ * @return A logically equivalent representation of the raw header content
+ */
+ public static IfNoneMatch fromHeader(String rawValue) {
+ // parse null header as an empty header
+ if (rawValue == null) {
+ return EMPTY;
+ }
+
+ rawValue = rawValue.trim();
+ if (rawValue.equals(WILDCARD_HEADER_VALUE)) {
+ return WILDCARD;
+ } else {
+
+ // ensure entire header matches expected pattern
+ if (!IF_NONE_MATCH_PATTERN.matcher(rawValue).matches()) {
+ throw new IllegalArgumentException("Invalid If-None-Match header: " +
rawValue);
+ }
+
+ // extract out ETags using the capture group
+ List<String> etags =
+
ETAG_PATTERN.matcher(rawValue).results().map(MatchResult::group).toList();
+
+ return new IfNoneMatch(etags);
+ }
+ }
+
+ /**
+ * If any contained ETag matches the provided etag or the header is a
wildcard. Only matches weak
+ * eTags to weak eTags and strong eTags to strong eTags.
+ *
+ * @param eTag the etag to compare against.
+ * @return true if any of the contained ETags match the provided etag
+ */
+ public boolean anyMatch(String eTag) {
+ if (this.isWildcard) return true;
+ return eTags.contains(eTag);
+ }
+}
diff --git
a/service/common/src/test/java/org/apache/polaris/service/http/IfNoneMatchTest.java
b/service/common/src/test/java/org/apache/polaris/service/http/IfNoneMatchTest.java
new file mode 100644
index 000000000..8e57d02da
--- /dev/null
+++
b/service/common/src/test/java/org/apache/polaris/service/http/IfNoneMatchTest.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.http;
+
+import java.util.List;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class IfNoneMatchTest {
+
+ @Test
+ public void validSingleETag() {
+ String header = "W/\"value\"";
+ IfNoneMatch ifNoneMatch = IfNoneMatch.fromHeader(header);
+
+ String parsedETag = ifNoneMatch.eTags().getFirst();
+
+ Assertions.assertEquals(header, parsedETag);
+ }
+
+ @Test
+ public void validMultipleETags() {
+ String etagValue1 = "W/\"etag1\"";
+ String etagValue2 = "W/\"etag2,with,comma\"";
+ String etagValue3 = "W/\"etag3\"";
+
+ String header = etagValue1 + ", " + etagValue2 + ", " + etagValue3;
+ IfNoneMatch ifNoneMatch = IfNoneMatch.fromHeader(header);
+
+ Assertions.assertEquals(3, ifNoneMatch.eTags().size());
+
+ String etag1 = ifNoneMatch.eTags().get(0);
+ String etag2 = ifNoneMatch.eTags().get(1);
+ String etag3 = ifNoneMatch.eTags().get(2);
+
+ Assertions.assertEquals(etagValue1, etag1);
+ Assertions.assertEquals(etagValue2, etag2);
+ Assertions.assertEquals(etagValue3, etag3);
+ }
+
+ @Test
+ public void validWildcardIfNoneMatch() {
+ IfNoneMatch ifNoneMatch = IfNoneMatch.fromHeader("*");
+ Assertions.assertTrue(ifNoneMatch.isWildcard());
+ Assertions.assertTrue(ifNoneMatch.eTags().isEmpty());
+ }
+
+ @Test
+ public void nullIfNoneMatchIsValidAndMapsToEmptyETags() {
+ // this test is important because we may not know what the representation
of
+ // the header will be if it is not provided. If it is null, then we should
default
+ // to an empty list of etags, as that has no footprint for logical
decisions to be made
+ IfNoneMatch nullIfNoneMatch = IfNoneMatch.fromHeader(null);
+ Assertions.assertTrue(nullIfNoneMatch.eTags().isEmpty());
+ }
+
+ @Test
+ public void invalidETagThrowsException() {
+ String header = "wrong_value";
+ Assertions.assertThrows(IllegalArgumentException.class, () ->
IfNoneMatch.fromHeader(header));
+ }
+
+ @Test
+ public void etagsMatch() {
+ String weakETag = "W/\"weak\"";
+ String strongETag = "\"strong\"";
+ IfNoneMatch ifNoneMatch = IfNoneMatch.fromHeader("W/\"weak\", \"strong\"");
+ Assertions.assertTrue(ifNoneMatch.anyMatch(weakETag));
+ Assertions.assertTrue(ifNoneMatch.anyMatch(strongETag));
+ }
+
+ @Test
+ public void weakETagOnlyMatchesWeak() {
+ String weakETag = "W/\"etag\"";
+ IfNoneMatch ifNoneMatch = IfNoneMatch.fromHeader("\"etag\"");
+ Assertions.assertFalse(ifNoneMatch.anyMatch(weakETag));
+ }
+
+ @Test
+ public void strongETagOnlyMatchesStrong() {
+ String strongETag = "\"etag\"";
+ IfNoneMatch ifNoneMatch = IfNoneMatch.fromHeader("W/\"etag\"");
+ Assertions.assertFalse(ifNoneMatch.anyMatch(strongETag));
+ }
+
+ @Test
+ public void wildCardMatchesEverything() {
+ String strongETag = "\"etag\"";
+ String weakETag = "W/\"etag\"";
+ IfNoneMatch ifNoneMatch = IfNoneMatch.fromHeader("*");
+ Assertions.assertTrue(ifNoneMatch.anyMatch(strongETag));
+ Assertions.assertTrue(ifNoneMatch.anyMatch(weakETag));
+
+ IfNoneMatch canonicallyBuiltWildcard = IfNoneMatch.WILDCARD;
+ Assertions.assertTrue(canonicallyBuiltWildcard.anyMatch(strongETag));
+ Assertions.assertTrue(canonicallyBuiltWildcard.anyMatch(weakETag));
+ }
+
+ @Test
+ public void cantConstructHeaderWithWildcardAndNonEmptyETag() {
+ Assertions.assertThrows(
+ IllegalArgumentException.class, () -> new IfNoneMatch(true,
List.of("\"etag\"")));
+ }
+
+ @Test
+ public void cantConstructHeaderWithOneValidAndOneInvalidPart() {
+ Assertions.assertThrows(
+ IllegalArgumentException.class, () ->
IfNoneMatch.fromHeader("W/\"etag\", W/invalid-etag"));
+ }
+
+ @Test
+ public void invalidHeaderWithOnlyWhitespacesBetween() {
+ Assertions.assertThrows(
+ IllegalArgumentException.class, () ->
IfNoneMatch.fromHeader("W/\"etag\" \"valid-etag\""));
+ }
+}
