This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 59e2ce6103 [#14298] improvement(iceberg): Own lakehouse Iceberg
clients and remint GCS tokens (#14300)
59e2ce6103 is described below
commit 59e2ce6103baba566391255a767c7ad614ebf0a2
Author: MaSai <[email protected]>
AuthorDate: Thu Oct 29 15:05:55 2026 +0800
[#14298] improvement(iceberg): Own lakehouse Iceberg clients and remint GCS
tokens (#14300)
### What changes were proposed in this pull request?
Keep metalake `lakehouse-iceberg` on its own Iceberg client, separate
from IRC. Add `IcebergBackend` so Hive, JDBC, and REST catalogs each own
a wrapper on the lakehouse class loader; before use it remints minted
GCS OAuth2 tokens and rebuilds the wrapper when
`gcs.oauth2.token-expires-at` is within 5 minutes.
- Add `IcebergBackend` (wrapper / helper / views + GCS remint)
- Route `IcebergCatalogOperations` through `backend.wrapper()` /
`helper()` / `views()`
- Do not borrow auxiliary IRC wrappers; REST still uses HTTP
`RESTCatalog` against `uri`
- Leave `CatalogManager` and IRC cache unchanged
### Why are the changes needed?
Lakehouse previously kept a long-lived `IcebergCatalogWrapper`. Minted
GCS OAuth2 tokens (`gcs.oauth2.token-expires-at`) could expire while
catalog ops stayed in `catalogCache`, so later metalake / TMS calls
failed.
Fixes #14298
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
- `TestIcebergGcsTokenRefresh`
- `TestIcebergEmbeddedRestBackend`
- `TestIcebergCatalog` / `TestIcebergCatalogOperations`
---------
Co-authored-by: Cursor <[email protected]>
---
.../catalog/lakehouse/iceberg/IcebergBackend.java | 158 +++++++++++++++++++++
.../iceberg/IcebergCatalogOperations.java | 131 +++++++----------
.../lakehouse/iceberg/TestIcebergCatalog.java | 2 +-
.../iceberg/TestIcebergCatalogOperations.java | 94 +++++-------
.../iceberg/TestIcebergEmbeddedRestBackend.java | 39 +++++
.../iceberg/TestIcebergGcsTokenRefresh.java | 66 +++++++++
6 files changed, 349 insertions(+), 141 deletions(-)
diff --git
a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergBackend.java
b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergBackend.java
new file mode 100644
index 0000000000..1ec59bceaa
--- /dev/null
+++
b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergBackend.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.catalog.lakehouse.iceberg;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import org.apache.commons.lang3.StringUtils;
+import
org.apache.gravitino.catalog.lakehouse.iceberg.ops.IcebergCatalogWrapperHelper;
+import org.apache.gravitino.iceberg.common.IcebergConfig;
+import org.apache.gravitino.iceberg.common.authentication.AuthenticationConfig;
+import org.apache.gravitino.iceberg.common.authentication.SupportsKerberos;
+import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper;
+import
org.apache.gravitino.iceberg.common.ops.KerberosAwareIcebergCatalogProxy;
+import org.apache.gravitino.iceberg.common.utils.IcebergCatalogUtil;
+import org.apache.iceberg.exceptions.NoSuchWarehouseException;
+import org.apache.iceberg.exceptions.RESTException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Owns the Iceberg client for one lakehouse catalog and remints minted GCS
OAuth2 tokens before
+ * use. {@link IcebergCatalogOperations} stays focused on Gravitino API
mapping.
+ *
+ * <p>Hive, JDBC, and REST clients are created on this class loader. REST
opens an HTTP connection
+ * to {@code uri}. Owned clients remint GCS tokens before {@code
gcs.oauth2.token-expires-at}.
+ */
+final class IcebergBackend implements AutoCloseable {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(IcebergBackend.class);
+
+ private static final long GCS_TOKEN_REFRESH_BUFFER_MS =
TimeUnit.MINUTES.toMillis(5);
+
+ private final Map<String, String> catalogConf;
+
+ private IcebergCatalogWrapper wrapper;
+ private IcebergCatalogWrapperHelper helper;
+ private IcebergViewCatalogOperations views;
+
+ private IcebergBackend(Map<String, String> catalogConf) {
+ this.catalogConf = catalogConf;
+ }
+
+ /**
+ * Creates a backend that owns its Iceberg client and remints GCS tokens.
+ *
+ * @param catalogConf resolved Iceberg catalog configuration
+ * @return backend that owns its client
+ */
+ static IcebergBackend create(Map<String, String> catalogConf) {
+ IcebergBackend backend = new IcebergBackend(catalogConf);
+ backend.reload();
+ return backend;
+ }
+
+ synchronized IcebergCatalogWrapper wrapper() {
+ refreshGcsTokenIfNeeded();
+ return wrapper;
+ }
+
+ synchronized IcebergCatalogWrapperHelper helper() {
+ refreshGcsTokenIfNeeded();
+ return helper;
+ }
+
+ synchronized IcebergViewCatalogOperations views() {
+ refreshGcsTokenIfNeeded();
+ return views;
+ }
+
+ private void refreshGcsTokenIfNeeded() {
+ if (wrapper == null) {
+ return;
+ }
+ String expiresAt =
+ wrapper
+ .getIcebergConfig()
+ .getAllConfig()
+ .get(IcebergConstants.ICEBERG_GCS_OAUTH2_TOKEN_EXPIRES_AT);
+ if (StringUtils.isBlank(expiresAt)) {
+ return;
+ }
+ try {
+ if (Long.parseLong(expiresAt) - GCS_TOKEN_REFRESH_BUFFER_MS >
System.currentTimeMillis()) {
+ return;
+ }
+ } catch (NumberFormatException e) {
+ LOG.warn("Invalid {}: {}",
IcebergConstants.ICEBERG_GCS_OAUTH2_TOKEN_EXPIRES_AT, expiresAt);
+ return;
+ }
+ IcebergCatalogWrapper previous = wrapper;
+ reload();
+ closeQuietly(previous);
+ }
+
+ private void reload() {
+ Map<String, String> conf = new HashMap<>(catalogConf);
+ IcebergConfig icebergConfig =
+ IcebergCatalogUtil.withGcsServiceAccountCredentials(new
IcebergConfig(conf));
+ try {
+ IcebergCatalogWrapper rawWrapper = new
IcebergCatalogWrapper(icebergConfig);
+ AuthenticationConfig authenticationConfig = new
AuthenticationConfig(catalogConf);
+ this.wrapper =
+ authenticationConfig.isKerberosAuth()
+ && rawWrapper.getCatalog() instanceof SupportsKerberos
+ ? new
KerberosAwareIcebergCatalogProxy(rawWrapper).getProxy(icebergConfig)
+ : rawWrapper;
+ this.helper = new IcebergCatalogWrapperHelper(wrapper.getCatalog());
+ this.views = new IcebergViewCatalogOperations(wrapper);
+ } catch (NoSuchWarehouseException e) {
+ throw new IllegalArgumentException(
+ String.format(
+ "The 'warehouse' value '%s' could not be resolved by the Iceberg
REST server. On "
+ + "the REST backend 'warehouse' selects a catalog by name on
the remote server "
+ + "and is not a storage location; remove 'warehouse' to use
the server's default "
+ + "catalog, or set it to a catalog name/identifier that the
server recognizes.",
+ icebergConfig.get(IcebergConfig.CATALOG_WAREHOUSE)),
+ e);
+ } catch (RESTException e) {
+ throw IcebergCatalogOperations.handleRestException(e);
+ }
+ }
+
+ @Override
+ public synchronized void close() {
+ closeQuietly(wrapper);
+ wrapper = null;
+ helper = null;
+ views = null;
+ }
+
+ private static void closeQuietly(IcebergCatalogWrapper catalogWrapper) {
+ if (catalogWrapper == null) {
+ return;
+ }
+ try {
+ catalogWrapper.close();
+ } catch (Exception e) {
+ LOG.warn("Failed to close Iceberg catalog wrapper", e);
+ }
+ }
+}
diff --git
a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java
b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java
index 79eec2fd8d..84b055c92a 100644
---
a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java
+++
b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java
@@ -52,12 +52,7 @@ import
org.apache.gravitino.exceptions.NonEmptySchemaException;
import org.apache.gravitino.exceptions.SchemaAlreadyExistsException;
import org.apache.gravitino.exceptions.TableAlreadyExistsException;
import org.apache.gravitino.exceptions.ViewAlreadyExistsException;
-import org.apache.gravitino.iceberg.common.IcebergConfig;
-import org.apache.gravitino.iceberg.common.authentication.AuthenticationConfig;
-import org.apache.gravitino.iceberg.common.authentication.SupportsKerberos;
-import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper;
import
org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper.IcebergTableChange;
-import
org.apache.gravitino.iceberg.common.ops.KerberosAwareIcebergCatalogProxy;
import org.apache.gravitino.iceberg.common.utils.IcebergCatalogUtil;
import org.apache.gravitino.iceberg.common.utils.IcebergIdentifierUtils;
import org.apache.gravitino.meta.AuditInfo;
@@ -81,7 +76,6 @@ import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.exceptions.AlreadyExistsException;
import org.apache.iceberg.exceptions.NamespaceNotEmptyException;
import org.apache.iceberg.exceptions.NoSuchNamespaceException;
-import org.apache.iceberg.exceptions.NoSuchWarehouseException;
import org.apache.iceberg.exceptions.RESTException;
import org.apache.iceberg.exceptions.ServiceUnavailableException;
import org.apache.iceberg.rest.requests.RenameTableRequest;
@@ -101,11 +95,9 @@ public class IcebergCatalogOperations
public static final Logger LOG =
LoggerFactory.getLogger(IcebergCatalogOperations.class);
- @VisibleForTesting IcebergCatalogWrapper icebergCatalogWrapper;
+ @VisibleForTesting IcebergBackend backend;
@VisibleForTesting @Nullable String catalogUuid;
- private IcebergCatalogWrapperHelper icebergCatalogWrapperHelper;
- private IcebergViewCatalogOperations icebergViewCatalogOperations;
/**
* Initializes the Iceberg catalog operations with the provided
configuration.
@@ -130,33 +122,7 @@ public class IcebergCatalogOperations
resultConf.putAll(gravitinoConfig);
this.catalogUuid = info.id().toString();
resultConf.put(IcebergConstants.CATALOG_UUID, catalogUuid);
- IcebergConfig icebergConfig = new IcebergConfig(resultConf);
-
- IcebergCatalogWrapper rawWrapper = new
IcebergCatalogWrapper(icebergConfig);
-
- try {
- AuthenticationConfig authenticationConfig = new
AuthenticationConfig(resultConf);
- this.icebergCatalogWrapper =
- authenticationConfig.isKerberosAuth()
- && rawWrapper.getCatalog() instanceof SupportsKerberos
- ? new
KerberosAwareIcebergCatalogProxy(rawWrapper).getProxy(icebergConfig)
- : rawWrapper;
- this.icebergCatalogWrapperHelper =
- new IcebergCatalogWrapperHelper(icebergCatalogWrapper.getCatalog());
- this.icebergViewCatalogOperations = new
IcebergViewCatalogOperations(icebergCatalogWrapper);
- } catch (NoSuchWarehouseException e) {
- // A reachable server rejecting the `warehouse` selector is a user
error. See issue #11943.
- throw new IllegalArgumentException(
- String.format(
- "The 'warehouse' value '%s' could not be resolved by the Iceberg
REST server. On "
- + "the REST backend 'warehouse' selects a catalog by name on
the remote server "
- + "and is not a storage location; remove 'warehouse' to use
the server's default "
- + "catalog, or set it to a catalog name/identifier that the
server recognizes.",
- icebergConfig.get(IcebergConfig.CATALOG_WAREHOUSE)),
- e);
- } catch (RESTException e) {
- throw handleRestException(e);
- }
+ this.backend = IcebergBackend.create(resultConf);
}
@Override
@@ -182,12 +148,8 @@ public class IcebergCatalogOperations
/** Closes the Iceberg catalog and releases the associated client pool. */
@Override
public void close() {
- if (null != icebergCatalogWrapper) {
- try {
- icebergCatalogWrapper.close();
- } catch (Exception e) {
- LOG.warn("Failed to close Iceberg catalog", e);
- }
+ if (backend != null) {
+ backend.close();
}
}
@@ -213,7 +175,7 @@ public class IcebergCatalogOperations
}
List<org.apache.iceberg.catalog.Namespace> namespaces =
- icebergCatalogWrapper.listNamespace(icebergParent).namespaces();
+ backend.wrapper().listNamespace(icebergParent).namespaces();
Namespace catalogNamespace = Namespace.of(namespace.level(0),
namespace.level(1));
return namespaces.stream()
@@ -260,9 +222,11 @@ public class IcebergCatalogOperations
.withCreateTime(Instant.now())
.build())
.build();
- icebergCatalogWrapper.createNamespace(
- createdSchema.toCreateRequest(
- IcebergCatalogWrapperHelper.getIcebergNamespace(ident.name())));
+ backend
+ .wrapper()
+ .createNamespace(
+ createdSchema.toCreateRequest(
+
IcebergCatalogWrapperHelper.getIcebergNamespace(ident.name())));
LOG.info(
"Created Iceberg schema (database) {} in Iceberg\ncurrentUser:{}
\ncomment: {} \nmetadata: {}",
ident.name(),
@@ -294,8 +258,9 @@ public class IcebergCatalogOperations
public IcebergSchema loadSchema(NameIdentifier ident) throws
NoSuchSchemaException {
try {
GetNamespaceResponse response =
- icebergCatalogWrapper.loadNamespace(
- IcebergCatalogWrapperHelper.getIcebergNamespace(ident.name()));
+ backend
+ .wrapper()
+
.loadNamespace(IcebergCatalogWrapperHelper.getIcebergNamespace(ident.name()));
IcebergSchema icebergSchema =
IcebergSchema.builder()
.withName(ident.name())
@@ -328,8 +293,9 @@ public class IcebergCatalogOperations
throws NoSuchSchemaException {
try {
GetNamespaceResponse response =
- icebergCatalogWrapper.loadNamespace(
- IcebergCatalogWrapperHelper.getIcebergNamespace(ident.name()));
+ backend
+ .wrapper()
+
.loadNamespace(IcebergCatalogWrapperHelper.getIcebergNamespace(ident.name()));
Map<String, String> metadata = response.properties();
List<String> removals = new ArrayList<>();
Map<String, String> updates = new HashMap<>();
@@ -363,9 +329,11 @@ public class IcebergCatalogOperations
UpdateNamespacePropertiesRequest updateNamespacePropertiesRequest =
UpdateNamespacePropertiesRequest.builder().updateAll(updates).removeAll(removals).build();
UpdateNamespacePropertiesResponse updateNamespacePropertiesResponse =
- icebergCatalogWrapper.updateNamespaceProperties(
- IcebergCatalogWrapperHelper.getIcebergNamespace(ident.name()),
- updateNamespacePropertiesRequest);
+ backend
+ .wrapper()
+ .updateNamespaceProperties(
+
IcebergCatalogWrapperHelper.getIcebergNamespace(ident.name()),
+ updateNamespacePropertiesRequest);
LOG.info(
"Altered Iceberg schema (database) {}. UpdateResponse:\n{}",
ident.name(),
@@ -391,8 +359,9 @@ public class IcebergCatalogOperations
public boolean dropSchema(NameIdentifier ident, boolean cascade) throws
NonEmptySchemaException {
Preconditions.checkArgument(!cascade, "Iceberg does not support cascading
delete operations.");
try {
- icebergCatalogWrapper.dropNamespace(
- IcebergCatalogWrapperHelper.getIcebergNamespace(ident.name()));
+ backend
+ .wrapper()
+
.dropNamespace(IcebergCatalogWrapperHelper.getIcebergNamespace(ident.name()));
LOG.info("Dropped Iceberg schema (database) {}", ident.name());
return true;
} catch (NamespaceNotEmptyException e) {
@@ -422,8 +391,7 @@ public class IcebergCatalogOperations
try {
ListTablesResponse listTablesResponse =
- icebergCatalogWrapper.listTable(
- IcebergCatalogWrapperHelper.getIcebergNamespace(namespace));
+
backend.wrapper().listTable(IcebergCatalogWrapperHelper.getIcebergNamespace(namespace));
return listTablesResponse.identifiers().stream()
.map(
tableIdentifier ->
@@ -446,8 +414,9 @@ public class IcebergCatalogOperations
try {
LoadTableResponse tableResponse =
- icebergCatalogWrapper.loadTable(
-
IcebergCatalogWrapperHelper.buildIcebergTableIdentifier(tableIdent));
+ backend
+ .wrapper()
+
.loadTable(IcebergCatalogWrapperHelper.buildIcebergTableIdentifier(tableIdent));
IcebergTable icebergTable =
IcebergTable.fromIcebergTable(tableResponse.tableMetadata(),
tableIdent.name());
@@ -496,9 +465,11 @@ public class IcebergCatalogOperations
try {
String[] levels = tableIdent.namespace().levels();
IcebergTableChange icebergTableChange =
- icebergCatalogWrapperHelper.buildIcebergTableChanges(
- NameIdentifier.of(levels[levels.length - 1], tableIdent.name()),
changes);
- LoadTableResponse loadTableResponse =
icebergCatalogWrapper.updateTable(icebergTableChange);
+ backend
+ .helper()
+ .buildIcebergTableChanges(
+ NameIdentifier.of(levels[levels.length - 1],
tableIdent.name()), changes);
+ LoadTableResponse loadTableResponse =
backend.wrapper().updateTable(icebergTableChange);
loadTableResponse.validate();
return IcebergTable.fromIcebergTable(loadTableResponse.tableMetadata(),
tableIdent.name());
} catch (org.apache.iceberg.exceptions.NoSuchTableException e) {
@@ -537,7 +508,7 @@ public class IcebergCatalogOperations
IcebergCatalogWrapperHelper.buildIcebergTableIdentifier(
destNamespace, renameTable.getNewName()))
.build();
- icebergCatalogWrapper.renameTable(renameTableRequest);
+ backend.wrapper().renameTable(renameTableRequest);
return loadTable(
NameIdentifier.of(ArrayUtils.add(destNamespace.levels(),
renameTable.getNewName())));
} catch (org.apache.iceberg.exceptions.NoSuchTableException e) {
@@ -554,8 +525,9 @@ public class IcebergCatalogOperations
@Override
public boolean dropTable(NameIdentifier tableIdent) {
try {
- icebergCatalogWrapper.dropTable(
- IcebergCatalogWrapperHelper.buildIcebergTableIdentifier(tableIdent));
+ backend
+ .wrapper()
+
.dropTable(IcebergCatalogWrapperHelper.buildIcebergTableIdentifier(tableIdent));
LOG.info("Dropped Iceberg table {}", tableIdent.name());
return true;
} catch (org.apache.iceberg.exceptions.NoSuchTableException e) {
@@ -631,9 +603,11 @@ public class IcebergCatalogOperations
.build();
LoadTableResponse loadTableResponse =
- icebergCatalogWrapper.createTable(
-
IcebergCatalogWrapperHelper.getIcebergNamespace(schemaIdent.name()),
- createdTable.toCreateTableRequest());
+ backend
+ .wrapper()
+ .createTable(
+
IcebergCatalogWrapperHelper.getIcebergNamespace(schemaIdent.name()),
+ createdTable.toCreateTableRequest());
loadTableResponse.validate();
LOG.info("Created Iceberg table {}", tableIdent.name());
@@ -654,7 +628,7 @@ public class IcebergCatalogOperations
public boolean purgeTable(NameIdentifier tableIdent) throws
UnsupportedOperationException {
try {
String schema =
NameIdentifier.of(tableIdent.namespace().levels()).name();
- icebergCatalogWrapper.purgeTable(TableIdentifier.of(schema,
tableIdent.name()));
+ backend.wrapper().purgeTable(TableIdentifier.of(schema,
tableIdent.name()));
LOG.info("Purge Iceberg table {}", tableIdent.name());
return true;
} catch (org.apache.iceberg.exceptions.NoSuchTableException e) {
@@ -673,7 +647,7 @@ public class IcebergCatalogOperations
@Override
public void testConnection(NameIdentifier catalogIdent) {
try {
-
icebergCatalogWrapper.listNamespace(IcebergCatalogWrapperHelper.getIcebergNamespace());
+
backend.wrapper().listNamespace(IcebergCatalogWrapperHelper.getIcebergNamespace());
} catch (Exception e) {
throw new ConnectionFailedException(
e, "Failed to run listNamespace on Iceberg catalog: %s",
e.getMessage());
@@ -691,7 +665,7 @@ public class IcebergCatalogOperations
*/
@Override
public View loadView(NameIdentifier ident) throws NoSuchViewException {
- return viewCatalogOperations().loadView(ident);
+ return backend.views().loadView(ident);
}
/**
@@ -702,7 +676,7 @@ public class IcebergCatalogOperations
*/
@Override
public boolean viewExists(NameIdentifier ident) {
- return viewCatalogOperations().viewExists(ident);
+ return backend.views().viewExists(ident);
}
/**
@@ -714,7 +688,7 @@ public class IcebergCatalogOperations
*/
@Override
public NameIdentifier[] listViews(Namespace namespace) throws
NoSuchSchemaException {
- return viewCatalogOperations().listViews(namespace);
+ return backend.views().listViews(namespace);
}
/**
@@ -744,7 +718,8 @@ public class IcebergCatalogOperations
String defaultSchema,
Map<String, String> properties)
throws NoSuchSchemaException, ViewAlreadyExistsException {
- return viewCatalogOperations()
+ return backend
+ .views()
.createView(
ident, comment, columns, representations, defaultCatalog,
defaultSchema, properties);
}
@@ -762,7 +737,7 @@ public class IcebergCatalogOperations
@Override
public View alterView(NameIdentifier ident, ViewChange... changes)
throws NoSuchViewException, IllegalArgumentException {
- return viewCatalogOperations().alterView(ident, changes);
+ return backend.views().alterView(ident, changes);
}
/**
@@ -773,7 +748,7 @@ public class IcebergCatalogOperations
*/
@Override
public boolean dropView(NameIdentifier ident) {
- return viewCatalogOperations().dropView(ident);
+ return backend.views().dropView(ident);
}
private static Distribution getIcebergDefaultDistribution(
@@ -789,8 +764,4 @@ public class IcebergCatalogOperations
private static String currentUser() {
return PrincipalUtils.getCurrentUserName();
}
-
- private IcebergViewCatalogOperations viewCatalogOperations() {
- return icebergViewCatalogOperations;
- }
}
diff --git
a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalog.java
b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalog.java
index ecd19aaf4c..a03c590d0b 100644
---
a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalog.java
+++
b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalog.java
@@ -134,7 +134,7 @@ public class TestIcebergCatalog {
Assertions.assertTrue(catalogOperations instanceof
IcebergCatalogOperations);
IcebergCatalogWrapper icebergCatalogWrapper =
- ((IcebergCatalogOperations) catalogOperations).icebergCatalogWrapper;
+ ((IcebergCatalogOperations) catalogOperations).backend.wrapper();
ListNamespacesResponse listNamespacesResponse =
icebergCatalogWrapper.listNamespace(org.apache.iceberg.catalog.Namespace.empty());
Assertions.assertTrue(listNamespacesResponse.namespaces().isEmpty());
diff --git
a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalogOperations.java
b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalogOperations.java
index 9b42c1849f..29ceb2b374 100644
---
a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalogOperations.java
+++
b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalogOperations.java
@@ -18,10 +18,6 @@
*/
package org.apache.gravitino.catalog.lakehouse.iceberg;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
import com.google.common.collect.ImmutableMap;
import java.util.Arrays;
import org.apache.gravitino.Catalog;
@@ -30,12 +26,11 @@ import org.apache.gravitino.Namespace;
import org.apache.gravitino.exceptions.ConnectionFailedException;
import org.apache.gravitino.exceptions.GravitinoRuntimeException;
import org.apache.gravitino.iceberg.common.IcebergConfig;
-import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper;
import org.apache.gravitino.iceberg.common.utils.IcebergCatalogUtil;
import org.apache.iceberg.exceptions.BadRequestException;
import org.apache.iceberg.exceptions.RESTException;
import org.apache.iceberg.exceptions.ServiceUnavailableException;
-import org.apache.iceberg.rest.responses.ListNamespacesResponse;
+import org.apache.iceberg.rest.requests.CreateNamespaceRequest;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -78,76 +73,49 @@ public class TestIcebergCatalogOperations {
Assertions.assertSame(badRequest,
IcebergCatalogOperations.handleRestException(badRequest));
}
- @Test
- public void testTestConnectionNonRestWarehouseHasNoHint() {
- IcebergCatalogWrapper mockWrapper = mock(IcebergCatalogWrapper.class);
- IcebergConfig config =
- new IcebergConfig(
- ImmutableMap.of(
- IcebergConstants.CATALOG_BACKEND, "hive",
- IcebergConstants.WAREHOUSE, "s3://warehouse/"));
- when(mockWrapper.getIcebergConfig()).thenReturn(config);
- when(mockWrapper.listNamespace(any())).thenThrow(new
RuntimeException("boom"));
-
- IcebergCatalogOperations catalogOperations = new
IcebergCatalogOperations();
- catalogOperations.icebergCatalogWrapper = mockWrapper;
-
- Exception exception =
- Assertions.assertThrows(
- ConnectionFailedException.class,
- () ->
- catalogOperations.testConnection(
- NameIdentifier.of(METALAKE, CATALOG),
- Catalog.Type.RELATIONAL,
- "iceberg",
- "comment",
- ImmutableMap.of()));
- Assertions.assertTrue(
- exception.getMessage().contains("Failed to run listNamespace on
Iceberg catalog"),
- exception.getMessage());
- Assertions.assertFalse(
- exception.getMessage().contains("selects a catalog by name"),
exception.getMessage());
- }
-
@Test
public void testListSchemasConvertsMultiLevelNamespacesToLogicalNames() {
- IcebergCatalogWrapper mockWrapper = mock(IcebergCatalogWrapper.class);
+ IcebergBackend backend = memoryBackend();
+ backend
+ .wrapper()
+ .createNamespace(
+ CreateNamespaceRequest.builder()
+ .withNamespace(org.apache.iceberg.catalog.Namespace.of("A",
"B", "C"))
+ .build());
IcebergCatalogOperations catalogOperations = new
IcebergCatalogOperations();
- catalogOperations.icebergCatalogWrapper = mockWrapper;
+ catalogOperations.backend = backend;
- org.apache.iceberg.catalog.Namespace flatNs =
org.apache.iceberg.catalog.Namespace.of("mydb");
- org.apache.iceberg.catalog.Namespace hierarchicalNs =
- org.apache.iceberg.catalog.Namespace.of("A", "B", "C");
- ListNamespacesResponse mockResponse =
- ListNamespacesResponse.builder().addAll(Arrays.asList(flatNs,
hierarchicalNs)).build();
- when(mockWrapper.listNamespace(any())).thenReturn(mockResponse);
+ NameIdentifier[] result =
catalogOperations.listSchemas(Namespace.of(METALAKE, CATALOG, "A:B"));
- NameIdentifier[] result =
catalogOperations.listSchemas(Namespace.of(METALAKE, CATALOG));
-
- Assertions.assertEquals(2, result.length);
- // Flat namespace stays as-is.
- Assertions.assertTrue(Arrays.stream(result).anyMatch(id ->
"mydb".equals(id.name())));
- // Multi-level Iceberg namespace is joined with the configured separator.
- Assertions.assertTrue(Arrays.stream(result).anyMatch(id ->
"A:B:C".equals(id.name())));
+ Assertions.assertEquals(1, result.length);
+ Assertions.assertEquals("A:B:C", result[0].name());
+ backend.close();
}
@Test
public void testListSchemasFlatOnlyReturnsUnchangedNames() {
- IcebergCatalogWrapper mockWrapper = mock(IcebergCatalogWrapper.class);
+ IcebergBackend backend = memoryBackend();
+ backend
+ .wrapper()
+ .createNamespace(
+ CreateNamespaceRequest.builder()
+ .withNamespace(org.apache.iceberg.catalog.Namespace.of("db1"))
+ .build());
+ backend
+ .wrapper()
+ .createNamespace(
+ CreateNamespaceRequest.builder()
+ .withNamespace(org.apache.iceberg.catalog.Namespace.of("db2"))
+ .build());
IcebergCatalogOperations catalogOperations = new
IcebergCatalogOperations();
- catalogOperations.icebergCatalogWrapper = mockWrapper;
-
- org.apache.iceberg.catalog.Namespace ns1 =
org.apache.iceberg.catalog.Namespace.of("db1");
- org.apache.iceberg.catalog.Namespace ns2 =
org.apache.iceberg.catalog.Namespace.of("db2");
- ListNamespacesResponse mockResponse =
- ListNamespacesResponse.builder().addAll(Arrays.asList(ns1,
ns2)).build();
- when(mockWrapper.listNamespace(any())).thenReturn(mockResponse);
+ catalogOperations.backend = backend;
NameIdentifier[] result =
catalogOperations.listSchemas(Namespace.of(METALAKE, CATALOG));
Assertions.assertEquals(2, result.length);
Assertions.assertTrue(Arrays.stream(result).anyMatch(id ->
"db1".equals(id.name())));
Assertions.assertTrue(Arrays.stream(result).anyMatch(id ->
"db2".equals(id.name())));
+ backend.close();
}
@Test
@@ -171,4 +139,10 @@ public class TestIcebergCatalogOperations {
Assertions.assertNotSame(original, replacement);
IcebergCatalogUtil.removeMemoryCatalog(catalogUuid);
}
+
+ private static IcebergBackend memoryBackend() {
+ return IcebergBackend.create(
+ ImmutableMap.of(
+ IcebergConstants.CATALOG_BACKEND, "memory",
IcebergConstants.WAREHOUSE, "/tmp/wh"));
+ }
}
diff --git
a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergEmbeddedRestBackend.java
b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergEmbeddedRestBackend.java
new file mode 100644
index 0000000000..4f5fc952f6
--- /dev/null
+++
b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergEmbeddedRestBackend.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.catalog.lakehouse.iceberg;
+
+import com.google.common.collect.ImmutableMap;
+import java.util.Map;
+import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class TestIcebergEmbeddedRestBackend {
+
+ @Test
+ void memoryCatalogOwnsItsClient() {
+ Map<String, String> conf =
+ ImmutableMap.of(
+ IcebergConstants.CATALOG_BACKEND, "memory",
IcebergConstants.WAREHOUSE, "/tmp/wh");
+ IcebergBackend backend = IcebergBackend.create(conf);
+ IcebergCatalogWrapper owned = backend.wrapper();
+ Assertions.assertNotNull(owned);
+ backend.close();
+ }
+}
diff --git
a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergGcsTokenRefresh.java
b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergGcsTokenRefresh.java
new file mode 100644
index 0000000000..70cc22d213
--- /dev/null
+++
b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/TestIcebergGcsTokenRefresh.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.catalog.lakehouse.iceberg;
+
+import com.google.common.collect.ImmutableMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class TestIcebergGcsTokenRefresh {
+
+ @Test
+ void reusesClientWhenTokenExpiryIsAbsent() {
+ IcebergBackend backend = IcebergBackend.create(memoryConfig(null));
+ IcebergCatalogWrapper wrapper = backend.wrapper();
+ Assertions.assertSame(wrapper, backend.wrapper());
+ backend.close();
+ }
+
+ @Test
+ void reusesClientWhenTokenIsFarFromExpiry() {
+ long expiresAt = System.currentTimeMillis() + TimeUnit.HOURS.toMillis(1);
+ IcebergBackend backend = IcebergBackend.create(memoryConfig(expiresAt));
+ IcebergCatalogWrapper wrapper = backend.wrapper();
+ Assertions.assertSame(wrapper, backend.wrapper());
+ backend.close();
+ }
+
+ @Test
+ void rebuildsClientInsideTheRefreshBuffer() {
+ long expiresAt = System.currentTimeMillis() + TimeUnit.MINUTES.toMillis(1);
+ IcebergBackend backend = IcebergBackend.create(memoryConfig(expiresAt));
+ IcebergCatalogWrapper first = backend.wrapper();
+ Assertions.assertNotSame(first, backend.wrapper());
+ backend.close();
+ }
+
+ private static Map<String, String> memoryConfig(Long expiresAt) {
+ ImmutableMap.Builder<String, String> builder =
+ ImmutableMap.<String, String>builder()
+ .put(IcebergConstants.CATALOG_BACKEND, "memory")
+ .put(IcebergConstants.WAREHOUSE, "/tmp/wh");
+ if (expiresAt != null) {
+ builder.put(IcebergConstants.ICEBERG_GCS_OAUTH2_TOKEN_EXPIRES_AT,
String.valueOf(expiresAt));
+ }
+ return builder.build();
+ }
+}