This is an automated email from the ASF dual-hosted git repository.
yufei pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new e32ef89bb Upgrade Iceberg to 1.8.1 (#1126)
e32ef89bb is described below
commit e32ef89bb97642f2ac9a4db82252a4fcf7aa0039
Author: Liam Bao <[email protected]>
AuthorDate: Mon Mar 31 22:53:42 2025 -0400
Upgrade Iceberg to 1.8.1 (#1126)
---
getting-started/spark/notebooks/SparkPolaris.ipynb | 2 +-
gradle/libs.versions.toml | 2 +-
.../it/test/PolarisApplicationIntegrationTest.java | 8 +--
.../it/test/PolarisRestCatalogIntegrationTest.java | 79 +++++++++++---------
.../PolarisRestCatalogViewIntegrationBase.java | 25 ++++++-
quarkus/service/build.gradle.kts | 5 ++
.../IcebergCatalogHandlerWrapperAuthzTest.java | 2 +-
.../quarkus/catalog/IcebergCatalogTest.java | 83 ++++++++++++++--------
.../quarkus/catalog/IcebergCatalogViewTest.java | 7 +-
.../it/QuarkusApplicationIntegrationTest.java | 4 ++
.../service/quarkus/task/TaskTestUtils.java | 5 +-
regtests/setup.sh | 2 +-
regtests/t_pyspark/src/iceberg_spark.py | 2 +-
regtests/t_spark_sql/ref/spark_sql_basic.sh.ref | 2 +-
regtests/t_spark_sql/ref/spark_sql_views.sh.ref | 2 +-
.../service/catalog/iceberg/IcebergCatalog.java | 5 +-
.../catalog/iceberg/IcebergCatalogAdapter.java | 4 ++
.../task/ManifestFileCleanupTaskHandler.java | 5 +-
.../service/exception/ExceptionMapperTest.java | 3 +-
site/content/in-dev/unreleased/quickstart.md | 2 +-
20 files changed, 163 insertions(+), 86 deletions(-)
diff --git a/getting-started/spark/notebooks/SparkPolaris.ipynb
b/getting-started/spark/notebooks/SparkPolaris.ipynb
index adb2f1a2c..48fff32ac 100644
--- a/getting-started/spark/notebooks/SparkPolaris.ipynb
+++ b/getting-started/spark/notebooks/SparkPolaris.ipynb
@@ -256,7 +256,7 @@
"\n",
"spark = (SparkSession.builder\n",
" .config(\"spark.sql.catalog.spark_catalog\",
\"org.apache.iceberg.spark.SparkSessionCatalog\")\n",
- " .config(\"spark.jars.packages\",
\"org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.7.1,org.apache.hadoop:hadoop-aws:3.4.0,software.amazon.awssdk:bundle:2.23.19,software.amazon.awssdk:url-connection-client:2.23.19\")\n",
+ " .config(\"spark.jars.packages\",
\"org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1,org.apache.hadoop:hadoop-aws:3.4.0,software.amazon.awssdk:bundle:2.23.19,software.amazon.awssdk:url-connection-client:2.23.19\")\n",
" .config('spark.sql.iceberg.vectorization.enabled', 'false')\n",
" \n",
" # Configure the 'polaris' catalog as an Iceberg rest catalog\n",
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index 87ec27921..3c87412f1 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -19,7 +19,7 @@
[versions]
hadoop = "3.4.1"
-iceberg = "1.7.1"
+iceberg = "1.8.1" # Ensure to update the iceberg version in regtests to keep
regtests up-to-date
quarkus = "3.19.4"
immutables = "2.10.1"
picocli = "4.7.6"
diff --git
a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java
b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java
index 2437601e0..99e736f79 100644
---
a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java
+++
b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java
@@ -454,8 +454,7 @@ public class PolarisApplicationIntegrationTest {
.assignUUID()
.addPartitionSpec(PartitionSpec.unpartitioned())
.addSortOrder(SortOrder.unsorted())
- .addSchema(
- new Schema(Types.NestedField.of(1, false, "col1",
Types.StringType.get())), 1)
+ .addSchema(new Schema(Types.NestedField.of(1, false, "col1",
Types.StringType.get())))
.build();
TableMetadataParser.write(tableMetadata,
fileIo.newOutputFile(metadataLocation));
@@ -498,7 +497,7 @@ public class PolarisApplicationIntegrationTest {
.assignUUID()
.addPartitionSpec(PartitionSpec.unpartitioned())
.addSortOrder(SortOrder.unsorted())
- .addSchema(new Schema(col1), 1)
+ .addSchema(new Schema(col1))
.build();
TableMetadataParser.write(tableMetadata,
fileIo.newOutputFile(metadataLocation));
@@ -546,8 +545,7 @@ public class PolarisApplicationIntegrationTest {
.assignUUID()
.addPartitionSpec(PartitionSpec.unpartitioned())
.addSortOrder(SortOrder.unsorted())
- .addSchema(
- new Schema(Types.NestedField.of(1, false, "col1",
Types.StringType.get())), 1)
+ .addSchema(new Schema(Types.NestedField.of(1, false, "col1",
Types.StringType.get())))
.build();
TableMetadataParser.write(tableMetadata,
fileIo.newOutputFile(metadataLocation));
diff --git
a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java
b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java
index a832553cf..83c9791b3 100644
---
a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java
+++
b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java
@@ -76,7 +76,6 @@ import org.apache.polaris.core.admin.model.TablePrivilege;
import org.apache.polaris.core.admin.model.ViewGrant;
import org.apache.polaris.core.admin.model.ViewPrivilege;
import org.apache.polaris.core.config.FeatureConfiguration;
-import org.apache.polaris.core.config.PolarisConfiguration;
import org.apache.polaris.core.entity.CatalogEntity;
import org.apache.polaris.core.entity.PolarisEntityConstants;
import org.apache.polaris.service.it.env.CatalogApi;
@@ -105,7 +104,7 @@ import org.junit.jupiter.api.io.TempDir;
* @implSpec @implSpec This test expects the server to be configured with the
following features
* configured:
* <ul>
- * <li>{@link
PolarisConfiguration#ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING}: {@code false}
+ * <li>{@link
FeatureConfiguration#ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING}: {@code false}
* </ul>
*/
@ExtendWith(PolarisIntegrationTestExtension.class)
@@ -118,16 +117,16 @@ public class PolarisRestCatalogIntegrationTest extends
CatalogTests<RESTCatalog>
private static URI externalCatalogBase;
protected static final String VIEW_QUERY = "select * from ns1.layer1_table";
- private static String principalRoleName;
private static ClientCredentials adminCredentials;
- private static PrincipalWithCredentials principalCredentials;
private static PolarisApiEndpoints endpoints;
private static PolarisClient client;
private static ManagementApi managementApi;
- private static CatalogApi catalogApi;
+ private PrincipalWithCredentials principalCredentials;
+ private CatalogApi catalogApi;
private RESTCatalog restCatalog;
private String currentCatalogName;
+ private Map<String, String> restCatalogConfig;
private final String catalogBaseLocation =
s3BucketBase + "/" + System.getenv("USER") + "/path/to/data";
@@ -159,10 +158,6 @@ public class PolarisRestCatalogIntegrationTest extends
CatalogTests<RESTCatalog>
endpoints = apiEndpoints;
client = polarisClient(endpoints);
managementApi = client.managementApi(credentials);
- String principalName = client.newEntityName("snowman-rest");
- principalRoleName = client.newEntityName("rest-admin");
- principalCredentials =
managementApi.createPrincipalWithRole(principalName, principalRoleName);
- catalogApi = client.catalogApi(principalCredentials);
URI testRootUri = IntegrationTestsHelper.getTemporaryDirectory(tempDir);
s3BucketBase = testRootUri.resolve("my-bucket");
externalCatalogBase = testRootUri.resolve("external-catalog");
@@ -175,10 +170,9 @@ public class PolarisRestCatalogIntegrationTest extends
CatalogTests<RESTCatalog>
@BeforeEach
public void before(TestInfo testInfo) {
- String principalName = "snowman-rest-" + UUID.randomUUID();
- principalRoleName = "rest-admin-" + UUID.randomUUID();
- PrincipalWithCredentials principalCredentials =
- managementApi.createPrincipalWithRole(principalName,
principalRoleName);
+ String principalName = client.newEntityName("snowman-rest");
+ String principalRoleName = client.newEntityName("rest-admin");
+ principalCredentials =
managementApi.createPrincipalWithRole(principalName, principalRoleName);
catalogApi = client.catalogApi(principalCredentials);
@@ -219,29 +213,26 @@ public class PolarisRestCatalogIntegrationTest extends
CatalogTests<RESTCatalog>
managementApi.createCatalog(principalRoleName, catalog);
- Optional<PolarisRestCatalogIntegrationTest.RestCatalogConfig>
restCatalogConfig =
+ restCatalogConfig =
testInfo
.getTestMethod()
- .flatMap(
- m ->
- Optional.ofNullable(
- m.getAnnotation(
-
PolarisRestCatalogIntegrationTest.RestCatalogConfig.class)));
- ImmutableMap.Builder<String, String> extraPropertiesBuilder =
ImmutableMap.builder();
- restCatalogConfig.ifPresent(
- config -> {
- for (int i = 0; i < config.value().length; i += 2) {
- extraPropertiesBuilder.put(config.value()[i], config.value()[i +
1]);
- }
- });
-
- restCatalog =
- IcebergHelper.restCatalog(
- client,
- endpoints,
- principalCredentials,
- currentCatalogName,
- extraPropertiesBuilder.build());
+ .map(m -> m.getAnnotation(RestCatalogConfig.class))
+ .map(RestCatalogConfig::value)
+ .map(
+ values -> {
+ if (values.length % 2 != 0) {
+ throw new IllegalArgumentException(
+ String.format("Missing value for config '%s'",
values[values.length - 1]));
+ }
+ Map<String, String> config = new HashMap<>();
+ for (int i = 0; i < values.length; i += 2) {
+ config.put(values[i], values[i + 1]);
+ }
+ return config;
+ })
+ .orElse(ImmutableMap.of());
+
+ restCatalog = initCatalog(currentCatalogName, ImmutableMap.of());
}
@AfterEach
@@ -254,6 +245,26 @@ public class PolarisRestCatalogIntegrationTest extends
CatalogTests<RESTCatalog>
return restCatalog;
}
+ /**
+ * Initialize a RESTCatalog for testing.
+ *
+ * @param catalogName this parameter is currently unused.
+ * @param additionalProperties additional properties to apply on top of the
default test settings
+ * @return a configured instance of RESTCatalog
+ */
+ @Override
+ protected RESTCatalog initCatalog(String catalogName, Map<String, String>
additionalProperties) {
+ ImmutableMap.Builder<String, String> extraPropertiesBuilder =
ImmutableMap.builder();
+ extraPropertiesBuilder.putAll(restCatalogConfig);
+ extraPropertiesBuilder.putAll(additionalProperties);
+ return IcebergHelper.restCatalog(
+ client,
+ endpoints,
+ principalCredentials,
+ currentCatalogName,
+ extraPropertiesBuilder.buildKeepingLast());
+ }
+
@Override
protected boolean requiresNamespaceCreate() {
return true;
diff --git
a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java
b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java
index 28c38c99e..8dc48837b 100644
---
a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java
+++
b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java
@@ -42,6 +42,8 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -113,7 +115,16 @@ public abstract class
PolarisRestCatalogViewIntegrationBase extends ViewCatalogT
managementApi.createCatalog(principalRoleName, catalog);
restCatalog =
- IcebergHelper.restCatalog(client, endpoints, principalCredentials,
catalogName, Map.of());
+ IcebergHelper.restCatalog(
+ client,
+ endpoints,
+ principalCredentials,
+ catalogName,
+ Map.of(
+ org.apache.iceberg.CatalogProperties.VIEW_DEFAULT_PREFIX +
"key1",
+ "catalog-default-key1",
+ org.apache.iceberg.CatalogProperties.VIEW_DEFAULT_PREFIX +
"key2",
+ "catalog-default-key2"));
}
@AfterEach
@@ -156,4 +167,16 @@ public abstract class
PolarisRestCatalogViewIntegrationBase extends ViewCatalogT
protected boolean overridesRequestedLocation() {
return true;
}
+
+ /** TODO: Unblock this test, see:
https://github.com/apache/polaris/issues/1273 */
+ @Override
+ @Test
+ @Disabled(
+ """
+ Disabled because the behavior is not applicable to Polaris.
+ To unblock, update this to expect an exception and add a
Polaris-specific test.
+ """)
+ public void createViewWithCustomMetadataLocation() {
+ super.createViewWithCustomMetadataLocation();
+ }
}
diff --git a/quarkus/service/build.gradle.kts b/quarkus/service/build.gradle.kts
index 2185c3ca1..ebc2c2917 100644
--- a/quarkus/service/build.gradle.kts
+++ b/quarkus/service/build.gradle.kts
@@ -23,6 +23,11 @@ plugins {
id("polaris-quarkus")
}
+configurations.all {
+ // exclude junit4 dependency for this module
+ exclude(group = "junit", module = "junit")
+}
+
dependencies {
implementation(project(":polaris-core"))
implementation(project(":polaris-api-management-service"))
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerWrapperAuthzTest.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerWrapperAuthzTest.java
index 527f23d0b..e028e5e3b 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerWrapperAuthzTest.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerWrapperAuthzTest.java
@@ -1721,7 +1721,7 @@ public class IcebergCatalogHandlerWrapperAuthzTest
extends PolarisAuthzTestBase
FileIO fileIO = CatalogUtil.loadFileIO(fileIoImpl, Map.of(), new
Configuration());
TableMetadata tableMetadata =
TableMetadata.buildFromEmpty()
- .addSchema(SCHEMA, SCHEMA.highestFieldId())
+ .addSchema(SCHEMA)
.setLocation(
String.format("%s/bucket/table/metadata/v1.metadata.json", storageLocation))
.addPartitionSpec(PartitionSpec.unpartitioned())
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java
index 8d520b9a9..4a5506fa2 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java
@@ -62,7 +62,6 @@ import org.apache.iceberg.TableMetadataParser;
import org.apache.iceberg.UpdateSchema;
import org.apache.iceberg.catalog.CatalogTests;
import org.apache.iceberg.catalog.Namespace;
-import org.apache.iceberg.catalog.SupportsNamespaces;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.exceptions.AlreadyExistsException;
import org.apache.iceberg.exceptions.BadRequestException;
@@ -128,6 +127,7 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.params.ParameterizedTest;
@@ -176,15 +176,12 @@ public abstract class IcebergCatalogTest extends
CatalogTests<IcebergCatalog> {
private IcebergCatalog catalog;
private CallContext callContext;
- private AwsStorageConfigInfo storageConfigModel;
- private StsClient stsClient;
private String realmName;
private PolarisMetaStoreManager metaStoreManager;
private PolarisCallContext polarisContext;
private PolarisAdminService adminService;
private PolarisEntityManager entityManager;
private FileIOFactory fileIOFactory;
- private AuthenticatedPolarisPrincipal authenticatedRoot;
private PolarisEntity catalogEntity;
private SecurityContext securityContext;
@@ -231,7 +228,8 @@ public abstract class IcebergCatalogTest extends
CatalogTests<IcebergCatalog> {
"root")
.getEntity()));
- authenticatedRoot = new AuthenticatedPolarisPrincipal(rootEntity,
Set.of());
+ AuthenticatedPolarisPrincipal authenticatedRoot =
+ new AuthenticatedPolarisPrincipal(rootEntity, Set.of());
securityContext = Mockito.mock(SecurityContext.class);
when(securityContext.getUserPrincipal()).thenReturn(authenticatedRoot);
@@ -245,7 +243,7 @@ public abstract class IcebergCatalogTest extends
CatalogTests<IcebergCatalog> {
new PolarisAuthorizerImpl(new PolarisConfigurationStore() {}));
String storageLocation = "s3://my-bucket/path/to/data";
- storageConfigModel =
+ AwsStorageConfigInfo storageConfigModel =
AwsStorageConfigInfo.builder()
.setRoleArn("arn:aws:iam::012345678901:role/jdoe")
.setExternalId("externalId")
@@ -266,10 +264,6 @@ public abstract class IcebergCatalogTest extends
CatalogTests<IcebergCatalog> {
.setStorageConfigurationInfo(storageConfigModel,
storageLocation)
.build());
- PolarisPassthroughResolutionView passthroughView =
- new PolarisPassthroughResolutionView(
- callContext, entityManager, securityContext, CATALOG_NAME);
- TaskExecutor taskExecutor = Mockito.mock();
RealmEntityManagerFactory realmEntityManagerFactory =
new RealmEntityManagerFactory(createMockMetaStoreManagerFactory());
this.fileIOFactory =
@@ -292,19 +286,7 @@ public abstract class IcebergCatalogTest extends
CatalogTests<IcebergCatalog> {
isA(AwsStorageConfigurationInfo.class)))
.thenReturn((PolarisStorageIntegration) storageIntegration);
- this.catalog =
- new IcebergCatalog(
- entityManager,
- metaStoreManager,
- callContext,
- passthroughView,
- securityContext,
- taskExecutor,
- fileIOFactory);
- this.catalog.initialize(
- CATALOG_NAME,
- ImmutableMap.of(
- CatalogProperties.FILE_IO_IMPL,
"org.apache.iceberg.inmemory.InMemoryFileIO"));
+ this.catalog = initCatalog("my-catalog", ImmutableMap.of());
}
@AfterEach
@@ -318,6 +300,37 @@ public abstract class IcebergCatalogTest extends
CatalogTests<IcebergCatalog> {
return catalog;
}
+ /**
+ * Initialize a IcebergCatalog for testing.
+ *
+ * @param catalogName this parameter is currently unused.
+ * @param additionalProperties additional properties to apply on top of the
default test settings
+ * @return a configured instance of IcebergCatalog
+ */
+ @Override
+ protected IcebergCatalog initCatalog(
+ String catalogName, Map<String, String> additionalProperties) {
+ PolarisPassthroughResolutionView passthroughView =
+ new PolarisPassthroughResolutionView(
+ callContext, entityManager, securityContext, CATALOG_NAME);
+ TaskExecutor taskExecutor = Mockito.mock();
+ IcebergCatalog icebergCatalog =
+ new IcebergCatalog(
+ entityManager,
+ metaStoreManager,
+ callContext,
+ passthroughView,
+ securityContext,
+ taskExecutor,
+ fileIOFactory);
+ ImmutableMap.Builder<String, String> propertiesBuilder =
+ ImmutableMap.<String, String>builder()
+ .put(CatalogProperties.FILE_IO_IMPL,
"org.apache.iceberg.inmemory.InMemoryFileIO")
+ .putAll(additionalProperties);
+ icebergCatalog.initialize(CATALOG_NAME,
propertiesBuilder.buildKeepingLast());
+ return icebergCatalog;
+ }
+
@Override
protected boolean requiresNamespaceCreate() {
return true;
@@ -373,6 +386,20 @@ public abstract class IcebergCatalogTest extends
CatalogTests<IcebergCatalog> {
};
}
+ /** TODO: Unblock this test, see:
https://github.com/apache/polaris/issues/1272 */
+ @Override
+ @Test
+ @Disabled(
+ """
+ Disabled because the behavior is not applicable to Polaris.
+ To unblock:
+ 1) Align Polaris behavior with the superclass by handling empty
namespaces the same way, or
+ 2) Modify this test to expect an exception and add a Polaris-specific
version.
+ """)
+ public void listNamespacesWithEmptyNamespace() {
+ super.listNamespacesWithEmptyNamespace();
+ }
+
@Test
public void testRenameTableMissingDestinationNamespace() {
Assumptions.assumeTrue(
@@ -697,7 +724,7 @@ public abstract class IcebergCatalogTest extends
CatalogTests<IcebergCatalog> {
TableMetadata.buildFromEmpty()
.assignUUID()
.setLocation(anotherTableLocation)
- .addSchema(SCHEMA, 4)
+ .addSchema(SCHEMA)
.addPartitionSpec(PartitionSpec.unpartitioned())
.addSortOrder(SortOrder.unsorted())
.build();
@@ -754,7 +781,7 @@ public abstract class IcebergCatalogTest extends
CatalogTests<IcebergCatalog> {
TableMetadata.buildFromEmpty()
.assignUUID()
.setLocation(anotherTableLocation)
- .addSchema(SCHEMA, 4)
+ .addSchema(SCHEMA)
.addPartitionSpec(PartitionSpec.unpartitioned())
.addSortOrder(SortOrder.unsorted())
.build();
@@ -831,7 +858,7 @@ public abstract class IcebergCatalogTest extends
CatalogTests<IcebergCatalog> {
TableMetadata.buildFromEmpty()
.assignUUID()
.setLocation(anotherTableLocation)
- .addSchema(SCHEMA, 4)
+ .addSchema(SCHEMA)
.addPartitionSpec(PartitionSpec.unpartitioned())
.addSortOrder(SortOrder.unsorted())
.build();
@@ -1401,7 +1428,7 @@ public abstract class IcebergCatalogTest extends
CatalogTests<IcebergCatalog> {
@Override
public void testDropTableWithPurge() {
if (this.requiresNamespaceCreate()) {
- ((SupportsNamespaces) catalog).createNamespace(NS);
+ catalog.createNamespace(NS);
}
Assertions.assertThatPredicate(catalog::tableExists)
@@ -1497,7 +1524,7 @@ public abstract class IcebergCatalogTest extends
CatalogTests<IcebergCatalog> {
CatalogProperties.FILE_IO_IMPL,
"org.apache.iceberg.inmemory.InMemoryFileIO"));
if (this.requiresNamespaceCreate()) {
- ((SupportsNamespaces) noPurgeCatalog).createNamespace(NS);
+ noPurgeCatalog.createNamespace(NS);
}
Assertions.assertThatPredicate(noPurgeCatalog::tableExists)
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java
index bea9aa310..7d817bec4 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java
@@ -199,7 +199,12 @@ public class IcebergCatalogViewTest extends
ViewCatalogTests<IcebergCatalog> {
this.catalog.initialize(
CATALOG_NAME,
ImmutableMap.of(
- CatalogProperties.FILE_IO_IMPL,
"org.apache.iceberg.inmemory.InMemoryFileIO"));
+ CatalogProperties.FILE_IO_IMPL,
+ "org.apache.iceberg.inmemory.InMemoryFileIO",
+ CatalogProperties.VIEW_DEFAULT_PREFIX + "key1",
+ "catalog-default-key1",
+ CatalogProperties.VIEW_DEFAULT_PREFIX + "key2",
+ "catalog-default-key2"));
}
@AfterEach
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusApplicationIntegrationTest.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusApplicationIntegrationTest.java
index ba1395ac8..9c72c9b06 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusApplicationIntegrationTest.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusApplicationIntegrationTest.java
@@ -36,6 +36,7 @@ import org.apache.iceberg.rest.ErrorHandlers;
import org.apache.iceberg.rest.HTTPClient;
import org.apache.iceberg.rest.RESTClient;
import org.apache.iceberg.rest.auth.AuthConfig;
+import org.apache.iceberg.rest.auth.AuthSession;
import org.apache.iceberg.rest.auth.OAuth2Util;
import org.apache.iceberg.rest.responses.OAuthTokenResponse;
import org.apache.polaris.service.auth.TokenBrokerFactory;
@@ -72,6 +73,7 @@ public class QuarkusApplicationIntegrationTest extends
PolarisApplicationIntegra
HTTPClient.builder(Map.of())
.withHeader(endpoints.realmHeaderName(), endpoints.realmId())
.uri(path)
+ .withAuthSession(AuthSession.EMPTY)
.build()) {
String credentialString =
clientCredentials.clientId() + ":" +
clientCredentials.clientSecret();
@@ -102,6 +104,7 @@ public class QuarkusApplicationIntegrationTest extends
PolarisApplicationIntegra
HTTPClient.builder(Map.of())
.withHeader(endpoints.realmHeaderName(), endpoints.realmId())
.uri(path)
+ .withAuthSession(AuthSession.EMPTY)
.build()) {
var response =
client.postForm(
@@ -142,6 +145,7 @@ public class QuarkusApplicationIntegrationTest extends
PolarisApplicationIntegra
HTTPClient.builder(Map.of())
.withHeader(endpoints.realmHeaderName(), endpoints.realmId())
.uri(path)
+ .withAuthSession(AuthSession.EMPTY)
.build()) {
var response =
client.postForm(
diff --git
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TaskTestUtils.java
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TaskTestUtils.java
index 9cd0ddd9e..d7538b923 100644
---
a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TaskTestUtils.java
+++
b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TaskTestUtils.java
@@ -100,8 +100,7 @@ public class TaskTestUtils {
tmBuilder
.setLocation("path/to/table")
.addSchema(
- new Schema(List.of(Types.NestedField.of(1, false, "field1",
Types.StringType.get()))),
- 1)
+ new Schema(List.of(Types.NestedField.of(1, false, "field1",
Types.StringType.get()))))
.addSortOrder(SortOrder.unsorted())
.assignUUID(UUID.randomUUID().toString())
.addPartitionSpec(PartitionSpec.unpartitioned());
@@ -110,7 +109,7 @@ public class TaskTestUtils {
for (Snapshot snapshot : snapshots) {
tmBuilder.addSnapshot(snapshot);
if (statisticsFiles != null) {
- tmBuilder.setStatistics(snapshot.snapshotId(),
statisticsFiles.get(statisticsFileIndex++));
+ tmBuilder.setStatistics(statisticsFiles.get(statisticsFileIndex++));
}
}
TableMetadata tableMetadata = tmBuilder.build();
diff --git a/regtests/setup.sh b/regtests/setup.sh
index 91a8bf5f6..e65e5a337 100755
--- a/regtests/setup.sh
+++ b/regtests/setup.sh
@@ -31,7 +31,7 @@ if [ -z "${SPARK_HOME}" ]; then
fi
SPARK_CONF="${SPARK_HOME}/conf/spark-defaults.conf"
DERBY_HOME="/tmp/derby"
-ICEBERG_VERSION="1.7.1"
+ICEBERG_VERSION="1.8.1"
export
PYTHONPATH="${SPARK_HOME}/python/:${SPARK_HOME}/python/lib/py4j-0.10.9.7-src.zip:$PYTHONPATH"
# Ensure binaries are downloaded locally
diff --git a/regtests/t_pyspark/src/iceberg_spark.py
b/regtests/t_pyspark/src/iceberg_spark.py
index 9b6a393d0..cd0c4ade5 100644
--- a/regtests/t_pyspark/src/iceberg_spark.py
+++ b/regtests/t_pyspark/src/iceberg_spark.py
@@ -72,7 +72,7 @@ class IcebergSparkSession:
"""Initial method for Iceberg Spark session. Creates a Spark session with
specified configs.
"""
packages = [
- "org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.7.1",
+ "org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1",
"org.apache.hadoop:hadoop-aws:3.4.0",
"software.amazon.awssdk:bundle:2.23.19",
"software.amazon.awssdk:url-connection-client:2.23.19",
diff --git a/regtests/t_spark_sql/ref/spark_sql_basic.sh.ref
b/regtests/t_spark_sql/ref/spark_sql_basic.sh.ref
index a23d1d941..4cd8ce0f8 100755
--- a/regtests/t_spark_sql/ref/spark_sql_basic.sh.ref
+++ b/regtests/t_spark_sql/ref/spark_sql_basic.sh.ref
@@ -1,4 +1,4 @@
-{"defaults":{"default-base-location":"file:///tmp/spark_sql_s3_catalog"},"overrides":{"prefix":"spark_sql_basic_catalog"},"endpoints":["GET
/v1/{prefix}/namespaces","GET /v1/{prefix}/namespaces/{namespace}","POST
/v1/{prefix}/namespaces","POST
/v1/{prefix}/namespaces/{namespace}/properties","DELETE
/v1/{prefix}/namespaces/{namespace}","GET
/v1/{prefix}/namespaces/{namespace}/tables","GET
/v1/{prefix}/namespaces/{namespace}/tables/{table}","POST
/v1/{prefix}/namespaces/{namespace}/tables" [...]
+{"defaults":{"default-base-location":"file:///tmp/spark_sql_s3_catalog"},"overrides":{"prefix":"spark_sql_basic_catalog"},"endpoints":["GET
/v1/{prefix}/namespaces","GET /v1/{prefix}/namespaces/{namespace}","HEAD
/v1/{prefix}/namespaces/{namespace}","POST /v1/{prefix}/namespaces","POST
/v1/{prefix}/namespaces/{namespace}/properties","DELETE
/v1/{prefix}/namespaces/{namespace}","GET
/v1/{prefix}/namespaces/{namespace}/tables","GET
/v1/{prefix}/namespaces/{namespace}/tables/{table}","HEAD [...]
Catalog created
spark-sql (default)> use polaris;
spark-sql ()> show namespaces;
diff --git a/regtests/t_spark_sql/ref/spark_sql_views.sh.ref
b/regtests/t_spark_sql/ref/spark_sql_views.sh.ref
index 8e4bff270..853c736db 100755
--- a/regtests/t_spark_sql/ref/spark_sql_views.sh.ref
+++ b/regtests/t_spark_sql/ref/spark_sql_views.sh.ref
@@ -1,4 +1,4 @@
-{"defaults":{"default-base-location":"file:///tmp/spark_sql_s3_catalog"},"overrides":{"prefix":"spark_sql_views_catalog"},"endpoints":["GET
/v1/{prefix}/namespaces","GET /v1/{prefix}/namespaces/{namespace}","POST
/v1/{prefix}/namespaces","POST
/v1/{prefix}/namespaces/{namespace}/properties","DELETE
/v1/{prefix}/namespaces/{namespace}","GET
/v1/{prefix}/namespaces/{namespace}/tables","GET
/v1/{prefix}/namespaces/{namespace}/tables/{table}","POST
/v1/{prefix}/namespaces/{namespace}/tables" [...]
+{"defaults":{"default-base-location":"file:///tmp/spark_sql_s3_catalog"},"overrides":{"prefix":"spark_sql_views_catalog"},"endpoints":["GET
/v1/{prefix}/namespaces","GET /v1/{prefix}/namespaces/{namespace}","HEAD
/v1/{prefix}/namespaces/{namespace}","POST /v1/{prefix}/namespaces","POST
/v1/{prefix}/namespaces/{namespace}/properties","DELETE
/v1/{prefix}/namespaces/{namespace}","GET
/v1/{prefix}/namespaces/{namespace}/tables","GET
/v1/{prefix}/namespaces/{namespace}/tables/{table}","HEAD [...]
Catalog created
spark-sql (default)> use polaris;
spark-sql ()> show namespaces;
diff --git
a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java
b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java
index a6c718151..3f9722f79 100644
---
a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java
+++
b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java
@@ -217,6 +217,10 @@ public class IcebergCatalog extends
BaseMetastoreViewCatalog
name,
this.catalogName);
+ // Ensure catalogProperties is assigned before calling metricsReporter()
for proper
+ // functionality.
+ catalogProperties = properties;
+
// Base location from catalogEntity is primary source of truth, otherwise
fall through
// to the same key from the properties map, and finally fall through to
WAREHOUSE_LOCATION.
String baseLocation =
@@ -263,7 +267,6 @@ public class IcebergCatalog extends BaseMetastoreViewCatalog
closeableGroup.addCloseable(metricsReporter());
closeableGroup.setSuppressCloseFailure(true);
- catalogProperties = properties;
tableDefaultProperties =
PropertyUtil.propertiesWithPrefix(properties,
CatalogProperties.TABLE_DEFAULT_PREFIX);
diff --git
a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java
b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java
index d88ea3a7e..7cb4f2f95 100644
---
a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java
+++
b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java
@@ -91,23 +91,27 @@ public class IcebergCatalogAdapter
ImmutableSet.<Endpoint>builder()
.add(Endpoint.V1_LIST_NAMESPACES)
.add(Endpoint.V1_LOAD_NAMESPACE)
+ .add(Endpoint.V1_NAMESPACE_EXISTS)
.add(Endpoint.V1_CREATE_NAMESPACE)
.add(Endpoint.V1_UPDATE_NAMESPACE)
.add(Endpoint.V1_DELETE_NAMESPACE)
.add(Endpoint.V1_LIST_TABLES)
.add(Endpoint.V1_LOAD_TABLE)
+ .add(Endpoint.V1_TABLE_EXISTS)
.add(Endpoint.V1_CREATE_TABLE)
.add(Endpoint.V1_UPDATE_TABLE)
.add(Endpoint.V1_DELETE_TABLE)
.add(Endpoint.V1_RENAME_TABLE)
.add(Endpoint.V1_REGISTER_TABLE)
.add(Endpoint.V1_REPORT_METRICS)
+ .add(Endpoint.V1_COMMIT_TRANSACTION)
.build();
private static final Set<Endpoint> VIEW_ENDPOINTS =
ImmutableSet.<Endpoint>builder()
.add(Endpoint.V1_LIST_VIEWS)
.add(Endpoint.V1_LOAD_VIEW)
+ .add(Endpoint.V1_VIEW_EXISTS)
.add(Endpoint.V1_CREATE_VIEW)
.add(Endpoint.V1_UPDATE_VIEW)
.add(Endpoint.V1_DELETE_VIEW)
diff --git
a/service/common/src/main/java/org/apache/polaris/service/task/ManifestFileCleanupTaskHandler.java
b/service/common/src/main/java/org/apache/polaris/service/task/ManifestFileCleanupTaskHandler.java
index 84e9683a2..3173dc25e 100644
---
a/service/common/src/main/java/org/apache/polaris/service/task/ManifestFileCleanupTaskHandler.java
+++
b/service/common/src/main/java/org/apache/polaris/service/task/ManifestFileCleanupTaskHandler.java
@@ -87,10 +87,7 @@ public class ManifestFileCleanupTaskHandler extends
FileCleanupTaskHandler {
StreamSupport.stream(
Spliterators.spliteratorUnknownSize(dataFiles.iterator(),
Spliterator.IMMUTABLE),
false)
- .map(
- file ->
- tryDelete(
- tableId, fileIO, manifestFile.path(),
file.path().toString(), null, 1))
+ .map(file -> tryDelete(tableId, fileIO, manifestFile.path(),
file.location(), null, 1))
.toList();
LOGGER.debug(
"Scheduled {} data files to be deleted from manifest {}",
diff --git
a/service/common/src/test/java/org/apache/polaris/service/exception/ExceptionMapperTest.java
b/service/common/src/test/java/org/apache/polaris/service/exception/ExceptionMapperTest.java
index 2a27f2f38..bc5f84c1d 100644
---
a/service/common/src/test/java/org/apache/polaris/service/exception/ExceptionMapperTest.java
+++
b/service/common/src/test/java/org/apache/polaris/service/exception/ExceptionMapperTest.java
@@ -42,7 +42,8 @@ public class ExceptionMapperTest {
@ParameterizedTest
@MethodSource("testFullExceptionIsLogged")
- public void testFullExceptionIsLogged(ExceptionMapper mapper, Exception
exception, Level level) {
+ public void testFullExceptionIsLogged(
+ ExceptionMapper<Exception> mapper, Exception exception, Level level) {
Logger logger = (Logger) LoggerFactory.getLogger(mapper.getClass());
ListAppender<ILoggingEvent> listAppender = new ListAppender<>();
listAppender.start();
diff --git a/site/content/in-dev/unreleased/quickstart.md
b/site/content/in-dev/unreleased/quickstart.md
index 9ab93c7fe..1b45ce91b 100644
--- a/site/content/in-dev/unreleased/quickstart.md
+++ b/site/content/in-dev/unreleased/quickstart.md
@@ -280,7 +280,7 @@ _Note: the credentials provided here are those for our
principal, not the root c
```shell
bin/spark-shell \
---packages
org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.7.1,org.apache.hadoop:hadoop-aws:3.4.0
\
+--packages
org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1,org.apache.hadoop:hadoop-aws:3.4.0
\
--conf
spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
\
--conf spark.sql.catalog.quickstart_catalog.warehouse=quickstart_catalog \
--conf
spark.sql.catalog.quickstart_catalog.header.X-Iceberg-Access-Delegation=vended-credentials
\
