This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new f37bd0d5d67 HIVE-28364: Iceberg: Upgrade iceberg version to 1.5.2
(Denys Kuzmenko, reviewed by Butao Zhang)
f37bd0d5d67 is described below
commit f37bd0d5d675e101009f3183bc7142d2d435cf79
Author: Denys Kuzmenko <[email protected]>
AuthorDate: Fri Jul 12 08:11:53 2024 +0200
HIVE-28364: Iceberg: Upgrade iceberg version to 1.5.2 (Denys Kuzmenko,
reviewed by Butao Zhang)
Closes #5338
---
iceberg/iceberg-catalog/pom.xml | 11 +
.../java/org/apache/iceberg/hive/HiveCatalog.java | 18 +-
.../org/apache/iceberg/hive/HiveClientPool.java | 2 +-
.../apache/iceberg/hive/HiveOperationsBase.java | 185 ++++++++++++++
.../apache/iceberg/hive/HiveTableOperations.java | 272 +++++++--------------
.../org/apache/iceberg/hive/MetastoreUtil.java | 17 +-
.../iceberg/hive/HiveCreateReplaceTableTest.java | 27 +-
.../iceberg/hive/HiveMetastoreExtension.java | 112 +++++++++
.../org/apache/iceberg/hive/HiveMetastoreTest.java | 86 -------
.../org/apache/iceberg/hive/HiveTableBaseTest.java | 32 ++-
.../org/apache/iceberg/hive/HiveTableTest.java | 44 ++--
.../apache/iceberg/hive/TestCachedClientPool.java | 26 +-
.../org/apache/iceberg/hive/TestHiveCatalog.java | 184 ++++++++------
.../apache/iceberg/hive/TestHiveCommitLocks.java | 59 ++++-
.../org/apache/iceberg/hive/TestHiveCommits.java | 57 ++++-
.../apache/iceberg/hive/TestHiveSchemaUtil.java | 6 +-
.../iceberg/hive/TestHiveTableConcurrency.java | 18 +-
.../iceberg/mr/hive/HiveIcebergMetaHook.java | 3 +-
.../java/org/apache/iceberg/SerializableTable.java | 49 +++-
iceberg/pom.xml | 8 +-
20 files changed, 802 insertions(+), 414 deletions(-)
diff --git a/iceberg/iceberg-catalog/pom.xml b/iceberg/iceberg-catalog/pom.xml
index d168744b345..02f617407b3 100644
--- a/iceberg/iceberg-catalog/pom.xml
+++ b/iceberg/iceberg-catalog/pom.xml
@@ -97,5 +97,16 @@
<artifactId>junit-vintage-engine</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.awaitility</groupId>
+ <artifactId>awaitility</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.iceberg</groupId>
+ <artifactId>iceberg-core</artifactId>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --git
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveCatalog.java
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveCatalog.java
index de859a50867..76935b7c74c 100644
---
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveCatalog.java
+++
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveCatalog.java
@@ -222,7 +222,7 @@ public class HiveCatalog extends BaseMetastoreCatalog
implements SupportsNamespa
try {
Table table = clients.run(client -> client.getTable(fromDatabase,
fromName));
- HiveTableOperations.validateTableIsIceberg(table, fullTableName(name,
from));
+ HiveOperationsBase.validateTableIsIceberg(table, fullTableName(name,
from));
table.setDbName(toDatabase);
table.setTableName(to.name());
@@ -237,8 +237,13 @@ public class HiveCatalog extends BaseMetastoreCatalog
implements SupportsNamespa
} catch (NoSuchObjectException e) {
throw new NoSuchTableException("Table does not exist: %s", from);
- } catch (AlreadyExistsException e) {
- throw new org.apache.iceberg.exceptions.AlreadyExistsException("Table
already exists: %s", to);
+ } catch (InvalidOperationException e) {
+ if (e.getMessage() != null && e.getMessage().contains(String.format("new
table %s already exists", to))) {
+ throw new org.apache.iceberg.exceptions.AlreadyExistsException(
+ "Table already exists: %s", to);
+ } else {
+ throw new RuntimeException("Failed to rename " + from + " to " + to,
e);
+ }
} catch (TException e) {
throw new RuntimeException("Failed to rename " + from + " to " + to, e);
@@ -271,8 +276,8 @@ public class HiveCatalog extends BaseMetastoreCatalog
implements SupportsNamespa
LOG.info("Created namespace: {}", namespace);
} catch (AlreadyExistsException e) {
- throw new org.apache.iceberg.exceptions.AlreadyExistsException(e,
"Namespace '%s' already exists!",
- namespace);
+ throw new org.apache.iceberg.exceptions.AlreadyExistsException(
+ e, "Namespace already exists: %s", namespace);
} catch (TException e) {
throw new RuntimeException("Failed to create namespace " + namespace + "
in Hive Metastore", e);
@@ -475,6 +480,9 @@ public class HiveCatalog extends BaseMetastoreCatalog
implements SupportsNamespa
return String.format("%s/%s", databaseData.getLocationUri(),
tableIdentifier.name());
}
+ } catch (NoSuchObjectException e) {
+ throw new NoSuchNamespaceException(
+ e, "Namespace does not exist: %s",
tableIdentifier.namespace().levels()[0]);
} catch (TException e) {
throw new RuntimeException(String.format("Metastore operation failed for
%s", tableIdentifier), e);
diff --git
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveClientPool.java
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveClientPool.java
index e9a3c53519b..604a99d8dfa 100644
---
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveClientPool.java
+++
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveClientPool.java
@@ -65,7 +65,7 @@ public class HiveClientPool extends
ClientPoolImpl<IMetaStoreClient, TException>
} catch (MetaException e) {
throw new RuntimeMetaException(e, "Failed to connect to Hive Metastore");
} catch (Throwable t) {
- if (t.getMessage().contains("Another instance of Derby may have already
booted")) {
+ if (t.getMessage() != null && t.getMessage().contains("Another instance
of Derby may have already booted")) {
throw new RuntimeMetaException(t, "Failed to start an embedded
metastore because embedded " +
"Derby supports only one client at a time. To fix this, use a
metastore that supports " +
"multiple clients.");
diff --git
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveOperationsBase.java
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveOperationsBase.java
new file mode 100644
index 00000000000..976c6bac2c5
--- /dev/null
+++
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveOperationsBase.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.hive;
+
+import java.util.Collections;
+import java.util.Map;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.ClientPool;
+import org.apache.iceberg.SchemaParser;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** All the HMS operations like table,view,materialized_view should implement
this. */
+public interface HiveOperationsBase {
+
+ Logger LOG = LoggerFactory.getLogger(HiveOperationsBase.class);
+ String HIVE_ICEBERG_STORAGE_HANDLER =
"org.apache.iceberg.mr.hive.HiveIcebergStorageHandler";
+
+ // The max size is based on HMS backend database. For Hive versions below
2.3, the max table parameter size is 4000
+ // characters, see https://issues.apache.org/jira/browse/HIVE-12274
+ // set to 0 to not expose Iceberg metadata in HMS Table properties.
+ String HIVE_TABLE_PROPERTY_MAX_SIZE = "iceberg.hive.table-property-max-size";
+ long HIVE_TABLE_PROPERTY_MAX_SIZE_DEFAULT = 32672;
+ String NO_LOCK_EXPECTED_KEY = "expected_parameter_key";
+ String NO_LOCK_EXPECTED_VALUE = "expected_parameter_value";
+
+ TableType tableType();
+
+ ClientPool<IMetaStoreClient, TException> metaClients();
+
+ long maxHiveTablePropertySize();
+
+ String database();
+
+ String table();
+
+ default Map<String, String> hmsEnvContext(String metadataLocation) {
+ return metadataLocation == null ? ImmutableMap.of() :
+ ImmutableMap.of(
+ NO_LOCK_EXPECTED_KEY,
+ BaseMetastoreTableOperations.METADATA_LOCATION_PROP,
+ NO_LOCK_EXPECTED_VALUE,
+ metadataLocation);
+ }
+
+ default boolean exposeInHmsProperties() {
+ return maxHiveTablePropertySize() > 0;
+ }
+
+ default void setSchema(TableMetadata metadata, Map<String, String>
parameters) {
+ parameters.remove(TableProperties.CURRENT_SCHEMA);
+ if (exposeInHmsProperties() && metadata.schema() != null) {
+ String schema = SchemaParser.toJson(metadata.schema());
+ setField(parameters, TableProperties.CURRENT_SCHEMA, schema);
+ }
+ }
+
+ default void setField(Map<String, String> parameters, String key, String
value) {
+ if (value.length() <= maxHiveTablePropertySize()) {
+ parameters.put(key, value);
+ } else {
+ LOG.warn(
+ "Not exposing {} in HMS since it exceeds {} characters", key,
maxHiveTablePropertySize());
+ }
+ }
+
+ static void validateTableIsIceberg(Table table, String fullName) {
+ String tableType =
table.getParameters().get(BaseMetastoreTableOperations.TABLE_TYPE_PROP);
+ NoSuchIcebergTableException.check(
+ tableType != null &&
tableType.equalsIgnoreCase(BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE),
+ "Not an iceberg table: %s (type=%s)", fullName, tableType);
+ }
+
+ static boolean isHiveIcebergStorageHandler(String storageHandler) {
+ try {
+ Class<?> storageHandlerClass = Class.forName(storageHandler);
+ return
Class.forName(HIVE_ICEBERG_STORAGE_HANDLER).isAssignableFrom(storageHandlerClass);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException("Error checking storage handler class", e);
+ }
+ }
+
+ default void persistTable(Table hmsTable, boolean updateHiveTable, String
metadataLocation)
+ throws TException, InterruptedException {
+ if (updateHiveTable) {
+ metaClients().run(
+ client -> {
+ MetastoreUtil.alterTable(
+ client, database(), table(), hmsTable,
hmsEnvContext(metadataLocation));
+ return null;
+ });
+ } else {
+ metaClients().run(
+ client -> {
+ client.createTable(hmsTable);
+ return null;
+ });
+ }
+ }
+
+ static StorageDescriptor storageDescriptor(TableMetadata metadata, boolean
hiveEngineEnabled) {
+ final StorageDescriptor storageDescriptor = new StorageDescriptor();
+ storageDescriptor.setCols(HiveSchemaUtil.convert(metadata.schema()));
+ storageDescriptor.setLocation(metadata.location());
+ SerDeInfo serDeInfo = new SerDeInfo();
+ serDeInfo.setParameters(Maps.newHashMap());
+ if (hiveEngineEnabled) {
+
storageDescriptor.setInputFormat("org.apache.iceberg.mr.hive.HiveIcebergInputFormat");
+
storageDescriptor.setOutputFormat("org.apache.iceberg.mr.hive.HiveIcebergOutputFormat");
+
serDeInfo.setSerializationLib("org.apache.iceberg.mr.hive.HiveIcebergSerDe");
+ } else {
+
storageDescriptor.setOutputFormat("org.apache.hadoop.mapred.FileOutputFormat");
+
storageDescriptor.setInputFormat("org.apache.hadoop.mapred.FileInputFormat");
+
serDeInfo.setSerializationLib("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe");
+ }
+ storageDescriptor.setSerdeInfo(serDeInfo);
+ return storageDescriptor;
+ }
+
+ static void cleanupMetadata(FileIO io, String commitStatus, String
metadataLocation) {
+ try {
+ if (commitStatus.equalsIgnoreCase("FAILURE")) {
+ // If we are sure the commit failed, clean up the uncommitted metadata
file
+ io.deleteFile(metadataLocation);
+ }
+ } catch (RuntimeException e) {
+ LOG.error("Failed to cleanup metadata file at {}", metadataLocation, e);
+ }
+ }
+
+ default Table newHmsTable(String hmsTableOwner) {
+ Preconditions.checkNotNull(hmsTableOwner, "'hmsOwner' parameter can't be
null");
+ final long currentTimeMillis = System.currentTimeMillis();
+
+ Table newTable = new Table(table(),
+ database(),
+ hmsTableOwner,
+ (int) currentTimeMillis / 1000,
+ (int) currentTimeMillis / 1000,
+ Integer.MAX_VALUE,
+ null,
+ Collections.emptyList(),
+ Maps.newHashMap(),
+ null,
+ null,
+ tableType().name());
+
+ if (tableType().equals(TableType.EXTERNAL_TABLE)) {
+ newTable
+ .getParameters()
+ .put("EXTERNAL", "TRUE"); // using the external table type also
requires this
+ }
+ return newTable;
+ }
+}
diff --git
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java
index aa6c1232e62..1249b11ed60 100644
---
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java
+++
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java
@@ -32,15 +32,12 @@ import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import
org.apache.hive.iceberg.com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.iceberg.BaseMetastoreTableOperations;
import org.apache.iceberg.ClientPool;
import org.apache.iceberg.PartitionSpecParser;
-import org.apache.iceberg.SchemaParser;
import org.apache.iceberg.Snapshot;
import org.apache.iceberg.SnapshotSummary;
import org.apache.iceberg.SortOrderParser;
@@ -49,13 +46,11 @@ import org.apache.iceberg.TableProperties;
import org.apache.iceberg.exceptions.AlreadyExistsException;
import org.apache.iceberg.exceptions.CommitFailedException;
import org.apache.iceberg.exceptions.CommitStateUnknownException;
-import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
import org.apache.iceberg.exceptions.NoSuchTableException;
import org.apache.iceberg.exceptions.ValidationException;
import org.apache.iceberg.hadoop.ConfigProperties;
import org.apache.iceberg.io.FileIO;
import
org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.BiMap;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableBiMap;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
@@ -72,26 +67,21 @@ import static org.apache.iceberg.TableProperties.GC_ENABLED;
* TODO we should be able to extract some more commonalities to
BaseMetastoreTableOperations to
* avoid code duplication between this class and Metacat Tables.
*/
-public class HiveTableOperations extends BaseMetastoreTableOperations {
+public class HiveTableOperations extends BaseMetastoreTableOperations
+ implements HiveOperationsBase {
private static final Logger LOG =
LoggerFactory.getLogger(HiveTableOperations.class);
- private static final String HIVE_ICEBERG_METADATA_REFRESH_MAX_RETRIES =
"iceberg.hive.metadata-refresh-max-retries";
- private static final int HIVE_ICEBERG_METADATA_REFRESH_MAX_RETRIES_DEFAULT =
2;
-
- // the max size is based on HMS backend database. For Hive versions below
2.3, the max table parameter size is 4000
- // characters, see https://issues.apache.org/jira/browse/HIVE-12274
- // set to 0 to not expose Iceberg metadata in HMS Table properties.
- private static final String HIVE_TABLE_PROPERTY_MAX_SIZE =
"iceberg.hive.table-property-max-size";
- private static final String NO_LOCK_EXPECTED_KEY = "expected_parameter_key";
- private static final String NO_LOCK_EXPECTED_VALUE =
"expected_parameter_value";
- private static final long HIVE_TABLE_PROPERTY_MAX_SIZE_DEFAULT = 32672;
- public static final String HIVE_ICEBERG_STORAGE_HANDLER =
"org.apache.iceberg.mr.hive.HiveIcebergStorageHandler";
+ private static final String HIVE_ICEBERG_METADATA_REFRESH_MAX_RETRIES =
+ "iceberg.hive.metadata-refresh-max-retries";
+ private static final int HIVE_ICEBERG_METADATA_REFRESH_MAX_RETRIES_DEFAULT =
2;
- private static final BiMap<String, String> ICEBERG_TO_HMS_TRANSLATION =
ImmutableBiMap.of(
- // gc.enabled in Iceberg and external.table.purge in Hive are meant to
do the same things but with different names
- GC_ENABLED, "external.table.purge",
- TableProperties.PARQUET_COMPRESSION, ParquetOutputFormat.COMPRESSION,
- TableProperties.PARQUET_ROW_GROUP_SIZE_BYTES,
ParquetOutputFormat.BLOCK_SIZE);
+ private static final BiMap<String, String> ICEBERG_TO_HMS_TRANSLATION =
+ ImmutableBiMap.of(
+ // gc.enabled in Iceberg and external.table.purge in Hive are meant
to do the same things
+ // but with different names
+ GC_ENABLED, "external.table.purge",
+ TableProperties.PARQUET_COMPRESSION, ParquetOutputFormat.COMPRESSION,
+ TableProperties.PARQUET_ROW_GROUP_SIZE_BYTES,
ParquetOutputFormat.BLOCK_SIZE);
/**
* Provides key translation where necessary between Iceberg and HMS props.
This translation is needed because some
@@ -120,8 +110,9 @@ public class HiveTableOperations extends
BaseMetastoreTableOperations {
private final FileIO fileIO;
private final ClientPool<IMetaStoreClient, TException> metaClients;
- protected HiveTableOperations(Configuration conf, ClientPool metaClients,
FileIO fileIO,
- String catalogName, String database, String
table) {
+ protected HiveTableOperations(
+ Configuration conf, ClientPool<IMetaStoreClient, TException>
metaClients, FileIO fileIO,
+ String catalogName, String database, String table) {
this.conf = conf;
this.metaClients = metaClients;
this.fileIO = fileIO;
@@ -130,8 +121,11 @@ public class HiveTableOperations extends
BaseMetastoreTableOperations {
this.database = database;
this.tableName = table;
this.metadataRefreshMaxRetries =
- conf.getInt(HIVE_ICEBERG_METADATA_REFRESH_MAX_RETRIES,
HIVE_ICEBERG_METADATA_REFRESH_MAX_RETRIES_DEFAULT);
- this.maxHiveTablePropertySize = conf.getLong(HIVE_TABLE_PROPERTY_MAX_SIZE,
HIVE_TABLE_PROPERTY_MAX_SIZE_DEFAULT);
+ conf.getInt(
+ HIVE_ICEBERG_METADATA_REFRESH_MAX_RETRIES,
+ HIVE_ICEBERG_METADATA_REFRESH_MAX_RETRIES_DEFAULT);
+ this.maxHiveTablePropertySize =
+ conf.getLong(HIVE_TABLE_PROPERTY_MAX_SIZE,
HIVE_TABLE_PROPERTY_MAX_SIZE_DEFAULT);
}
@Override
@@ -149,7 +143,7 @@ public class HiveTableOperations extends
BaseMetastoreTableOperations {
String metadataLocation = null;
try {
Table table = metaClients.run(client -> client.getTable(database,
tableName));
- validateTableIsIceberg(table, fullName);
+ HiveOperationsBase.validateTableIsIceberg(table, fullName);
metadataLocation = table.getParameters().get(METADATA_LOCATION_PROP);
@@ -159,7 +153,8 @@ public class HiveTableOperations extends
BaseMetastoreTableOperations {
}
} catch (TException e) {
- String errMsg = String.format("Failed to get table info from metastore
%s.%s", database, tableName);
+ String errMsg =
+ String.format("Failed to get table info from metastore %s.%s",
database, tableName);
throw new RuntimeException(errMsg, e);
} catch (InterruptedException e) {
@@ -181,7 +176,7 @@ public class HiveTableOperations extends
BaseMetastoreTableOperations {
CommitStatus commitStatus = CommitStatus.FAILURE;
boolean updateHiveTable = false;
- HiveLock lock = lockObject(metadata);
+ HiveLock lock = lockObject(base);
try {
lock.lock();
@@ -196,32 +191,36 @@ public class HiveTableOperations extends
BaseMetastoreTableOperations {
updateHiveTable = true;
LOG.debug("Committing existing table: {}", fullName);
} else {
- tbl = newHmsTable(metadata);
+ tbl = newHmsTable(metadata.property(HiveCatalog.HMS_TABLE_OWNER,
HiveHadoopUtil.currentUser()));
LOG.debug("Committing new table: {}", fullName);
}
- tbl.setSd(storageDescriptor(metadata, hiveEngineEnabled)); // set to
pickup any schema changes
+ tbl.setSd(HiveOperationsBase.storageDescriptor(metadata,
hiveEngineEnabled)); // set to pickup any schema changes
String metadataLocation =
tbl.getParameters().get(METADATA_LOCATION_PROP);
String baseMetadataLocation = base != null ? base.metadataFileLocation()
: null;
if (!Objects.equals(baseMetadataLocation, metadataLocation)) {
throw new CommitFailedException(
- "Base metadata location '%s' is not same as the current table
metadata location '%s' for %s.%s",
+ "Cannot commit: Base metadata location '%s' is not same as the
current table metadata location" +
+ " '%s' for %s.%s",
baseMetadataLocation, metadataLocation, database, tableName);
}
// get Iceberg props that have been removed
Set<String> removedProps = Collections.emptySet();
if (base != null) {
- removedProps = base.properties().keySet().stream()
- .filter(key -> !metadata.properties().containsKey(key))
- .collect(Collectors.toSet());
+ removedProps =
+ base.properties().keySet().stream()
+ .filter(key -> !metadata.properties().containsKey(key))
+ .collect(Collectors.toSet());
}
- Map<String, String> summary =
Optional.ofNullable(metadata.currentSnapshot())
- .map(Snapshot::summary)
- .orElseGet(ImmutableMap::of);
- setHmsTableParameters(newMetadataLocation, tbl, metadata, removedProps,
hiveEngineEnabled, summary);
+ Map<String, String> summary =
+ Optional.ofNullable(metadata.currentSnapshot())
+ .map(Snapshot::summary)
+ .orElseGet(ImmutableMap::of);
+ setHmsTableParameters(
+ newMetadataLocation, tbl, metadata, removedProps, hiveEngineEnabled,
summary);
if (!keepHiveStats) {
StatsSetupConst.setBasicStatsState(tbl.getParameters(),
StatsSetupConst.FALSE);
@@ -231,18 +230,18 @@ public class HiveTableOperations extends
BaseMetastoreTableOperations {
lock.ensureActive();
try {
persistTable(
- tbl, updateHiveTable, hiveLockEnabled(metadata, conf) ? null :
baseMetadataLocation);
+ tbl, updateHiveTable, hiveLockEnabled(base, conf) ? null :
baseMetadataLocation);
lock.ensureActive();
commitStatus = CommitStatus.SUCCESS;
} catch (LockException le) {
commitStatus = CommitStatus.UNKNOWN;
throw new CommitStateUnknownException(
- "Failed to heartbeat for hive lock while " +
- "committing changes. This can lead to a concurrent commit
attempt be able to overwrite this commit. " +
- "Please check the commit history. If you are running into this
issue, try reducing " +
- "iceberg.hive.lock-heartbeat-interval-ms.",
- le);
+ "Failed to heartbeat for hive lock while " +
+ "committing changes. This can lead to a concurrent commit
attempt be able to overwrite this commit. " +
+ "Please check the commit history. If you are running into this
issue, try reducing " +
+ "iceberg.hive.lock-heartbeat-interval-ms.",
+ le);
} catch (org.apache.hadoop.hive.metastore.api.AlreadyExistsException e) {
throw new AlreadyExistsException(e, "Table already exists: %s.%s",
database, tableName);
@@ -253,11 +252,10 @@ public class HiveTableOperations extends
BaseMetastoreTableOperations {
throw e;
} catch (Throwable e) {
- if (e.getMessage()
- .contains(
- "The table has been modified. The parameter value for key '" +
- HiveTableOperations.METADATA_LOCATION_PROP +
- "' is")) {
+ if (e.getMessage() != null && e.getMessage().contains(
+ "The table has been modified. The parameter value for key '" +
+ HiveTableOperations.METADATA_LOCATION_PROP +
+ "' is")) {
throw new CommitFailedException(
e, "The table %s.%s has been modified concurrently", database,
tableName);
}
@@ -265,8 +263,8 @@ public class HiveTableOperations extends
BaseMetastoreTableOperations {
if (e.getMessage() != null && e.getMessage().contains("Table/View
'HIVE_LOCKS' does not exist")) {
throw new RuntimeException(
"Failed to acquire locks from metastore because the underlying
metastore " +
- "table 'HIVE_LOCKS' does not exist. This can occur when
using an embedded metastore which does not " +
- "support transactions. To fix this use an alternative
metastore.",
+ "table 'HIVE_LOCKS' does not exist. This can occur when using
an embedded metastore which does not " +
+ "support transactions. To fix this use an alternative
metastore.",
e);
}
@@ -283,7 +281,8 @@ public class HiveTableOperations extends
BaseMetastoreTableOperations {
}
}
} catch (TException e) {
- throw new RuntimeException(String.format("Metastore operation failed for
%s.%s", database, tableName), e);
+ throw new RuntimeException(
+ String.format("Metastore operation failed for %s.%s", database,
tableName), e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
@@ -299,34 +298,6 @@ public class HiveTableOperations extends
BaseMetastoreTableOperations {
LOG.info("Committed to table {} with the new metadata location {}",
fullName, newMetadataLocation);
}
- @VisibleForTesting
- void persistTable(Table hmsTable, boolean updateHiveTable, String
expectedMetadataLocation)
- throws TException, InterruptedException {
- if (updateHiveTable) {
- metaClients.run(
- client -> {
- MetastoreUtil.alterTable(
- client,
- database,
- tableName,
- hmsTable,
- expectedMetadataLocation != null ?
- ImmutableMap.of(
- NO_LOCK_EXPECTED_KEY,
- METADATA_LOCATION_PROP,
- NO_LOCK_EXPECTED_VALUE,
- expectedMetadataLocation) :
- ImmutableMap.of());
- return null;
- });
- } else {
- metaClients.run(client -> {
- client.createTable(hmsTable);
- return null;
- });
- }
- }
-
@VisibleForTesting
Table loadHmsTable() throws TException, InterruptedException {
try {
@@ -337,43 +308,22 @@ public class HiveTableOperations extends
BaseMetastoreTableOperations {
}
}
- private Table newHmsTable(TableMetadata metadata) {
- Preconditions.checkNotNull(metadata, "'metadata' parameter can't be null");
- final long currentTimeMillis = System.currentTimeMillis();
-
- Table newTable = new Table(tableName,
- database,
- metadata.property(HiveCatalog.HMS_TABLE_OWNER,
HiveHadoopUtil.currentUser()),
- (int) currentTimeMillis / 1000,
- (int) currentTimeMillis / 1000,
- Integer.MAX_VALUE,
- null,
- Collections.emptyList(),
- Maps.newHashMap(),
- null,
- null,
- TableType.EXTERNAL_TABLE.toString());
-
- newTable.getParameters().put("EXTERNAL", "TRUE"); // using the external
table type also requires this
- return newTable;
- }
-
private void setHmsTableParameters(String newMetadataLocation, Table tbl,
TableMetadata metadata,
- Set<String> obsoleteProps, boolean
hiveEngineEnabled,
- Map<String, String> summary) {
+ Set<String> obsoleteProps, boolean hiveEngineEnabled,
+ Map<String, String> summary) {
Map<String, String> parameters = Optional.ofNullable(tbl.getParameters())
.orElseGet(Maps::newHashMap);
// push all Iceberg table properties into HMS
metadata.properties().entrySet().stream()
- .filter(entry ->
!entry.getKey().equalsIgnoreCase(HiveCatalog.HMS_TABLE_OWNER))
- .forEach(
- entry -> {
- String key = entry.getKey();
- // translate key names between Iceberg and HMS where needed
- String hmsKey = ICEBERG_TO_HMS_TRANSLATION.getOrDefault(key,
key);
- parameters.put(hmsKey, entry.getValue());
- });
+ .filter(entry ->
!entry.getKey().equalsIgnoreCase(HiveCatalog.HMS_TABLE_OWNER))
+ .forEach(
+ entry -> {
+ String key = entry.getKey();
+ // translate key names between Iceberg and HMS where needed
+ String hmsKey = ICEBERG_TO_HMS_TRANSLATION.getOrDefault(key,
key);
+ parameters.put(hmsKey, entry.getValue());
+ });
if (metadata.uuid() != null) {
parameters.put(TableProperties.UUID, metadata.uuid());
}
@@ -392,7 +342,7 @@ public class HiveTableOperations extends
BaseMetastoreTableOperations {
if (hiveEngineEnabled) {
String storageHandler =
parameters.get(hive_metastoreConstants.META_TABLE_STORAGE);
// Check if META_TABLE_STORAGE is not present or is not an instance of
ICEBERG_STORAGE_HANDLER
- if (storageHandler == null ||
!isHiveIcebergStorageHandler(storageHandler)) {
+ if (storageHandler == null ||
!HiveOperationsBase.isHiveIcebergStorageHandler(storageHandler)) {
parameters.put(hive_metastoreConstants.META_TABLE_STORAGE,
HIVE_ICEBERG_STORAGE_HANDLER);
}
} else {
@@ -426,8 +376,10 @@ public class HiveTableOperations extends
BaseMetastoreTableOperations {
Snapshot currentSnapshot = metadata.currentSnapshot();
if (exposeInHmsProperties() && currentSnapshot != null) {
- parameters.put(TableProperties.CURRENT_SNAPSHOT_ID,
String.valueOf(currentSnapshot.snapshotId()));
- parameters.put(TableProperties.CURRENT_SNAPSHOT_TIMESTAMP,
String.valueOf(currentSnapshot.timestampMillis()));
+ parameters.put(
+ TableProperties.CURRENT_SNAPSHOT_ID,
String.valueOf(currentSnapshot.snapshotId()));
+ parameters.put(
+ TableProperties.CURRENT_SNAPSHOT_TIMESTAMP,
String.valueOf(currentSnapshot.timestampMillis()));
setSnapshotSummary(parameters, currentSnapshot);
}
@@ -442,19 +394,11 @@ public class HiveTableOperations extends
BaseMetastoreTableOperations {
parameters.put(TableProperties.CURRENT_SNAPSHOT_SUMMARY, summary);
} else {
LOG.warn("Not exposing the current snapshot({}) summary in HMS since
it exceeds {} characters",
- currentSnapshot.snapshotId(), maxHiveTablePropertySize);
+ currentSnapshot.snapshotId(), maxHiveTablePropertySize);
}
} catch (JsonProcessingException e) {
- LOG.warn("Failed to convert current snapshot({}) summary to a json
string", currentSnapshot.snapshotId(), e);
- }
- }
-
- @VisibleForTesting
- void setSchema(TableMetadata metadata, Map<String, String> parameters) {
- parameters.remove(TableProperties.CURRENT_SCHEMA);
- if (exposeInHmsProperties() && metadata.schema() != null) {
- String schema = SchemaParser.toJson(metadata.schema());
- setField(parameters, TableProperties.CURRENT_SCHEMA, schema);
+ LOG.warn("Failed to convert current snapshot({}) summary to a json
string",
+ currentSnapshot.snapshotId(), e);
}
}
@@ -476,47 +420,35 @@ public class HiveTableOperations extends
BaseMetastoreTableOperations {
}
}
- private void setField(Map<String, String> parameters, String key, String
value) {
- if (value.length() <= maxHiveTablePropertySize) {
- parameters.put(key, value);
- } else {
- LOG.warn("Not exposing {} in HMS since it exceeds {} characters", key,
maxHiveTablePropertySize);
- }
+ @Override
+ public long maxHiveTablePropertySize() {
+ return maxHiveTablePropertySize;
}
- private boolean exposeInHmsProperties() {
- return maxHiveTablePropertySize > 0;
+ @Override
+ public String database() {
+ return database;
}
- private StorageDescriptor storageDescriptor(TableMetadata metadata, boolean
hiveEngineEnabled) {
+ @Override
+ public String table() {
+ return tableName;
+ }
- final StorageDescriptor storageDescriptor = new StorageDescriptor();
- storageDescriptor.setCols(HiveSchemaUtil.convert(metadata.schema()));
- storageDescriptor.setLocation(metadata.location());
- SerDeInfo serDeInfo = new SerDeInfo();
- serDeInfo.setParameters(Maps.newHashMap());
- if (hiveEngineEnabled) {
-
storageDescriptor.setInputFormat("org.apache.iceberg.mr.hive.HiveIcebergInputFormat");
-
storageDescriptor.setOutputFormat("org.apache.iceberg.mr.hive.HiveIcebergOutputFormat");
-
serDeInfo.setSerializationLib("org.apache.iceberg.mr.hive.HiveIcebergSerDe");
- } else {
-
storageDescriptor.setOutputFormat("org.apache.hadoop.mapred.FileOutputFormat");
-
storageDescriptor.setInputFormat("org.apache.hadoop.mapred.FileInputFormat");
-
serDeInfo.setSerializationLib("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe");
- }
- storageDescriptor.setSerdeInfo(serDeInfo);
- return storageDescriptor;
+ @Override
+ public TableType tableType() {
+ return TableType.EXTERNAL_TABLE;
+ }
+
+ @Override
+ public ClientPool<IMetaStoreClient, TException> metaClients() {
+ return metaClients;
}
private void cleanupMetadataAndUnlock(CommitStatus commitStatus, String
metadataLocation,
HiveLock lock) {
try {
- if (commitStatus == CommitStatus.FAILURE) {
- // If we are sure the commit failed, clean up the uncommitted metadata
file
- io().deleteFile(metadataLocation);
- }
- } catch (RuntimeException e) {
- LOG.error("Failed to cleanup metadata file at {}", metadataLocation, e);
+ HiveOperationsBase.cleanupMetadata(io(), commitStatus.name(),
metadataLocation);
} finally {
doUnlock(lock);
}
@@ -532,12 +464,6 @@ public class HiveTableOperations extends
BaseMetastoreTableOperations {
}
}
- static void validateTableIsIceberg(Table table, String fullName) {
- String tableType = table.getParameters().get(TABLE_TYPE_PROP);
- NoSuchIcebergTableException.check(tableType != null &&
tableType.equalsIgnoreCase(ICEBERG_TABLE_TYPE_VALUE),
- "Not an iceberg table: %s (type=%s)", fullName, tableType);
- }
-
/**
* Returns if the hive engine related values should be enabled on the table,
or not.
* <p>
@@ -558,7 +484,8 @@ public class HiveTableOperations extends
BaseMetastoreTableOperations {
return metadata.propertyAsBoolean(TableProperties.ENGINE_HIVE_ENABLED,
false);
}
- return conf.getBoolean(ConfigProperties.ENGINE_HIVE_ENABLED, true);
+ return conf.getBoolean(
+ ConfigProperties.ENGINE_HIVE_ENABLED, true);
}
/**
@@ -579,7 +506,7 @@ public class HiveTableOperations extends
BaseMetastoreTableOperations {
* @return if the hive engine related values should be enabled or not
*/
private static boolean hiveLockEnabled(TableMetadata metadata, Configuration
conf) {
- if (metadata.properties().get(TableProperties.HIVE_LOCK_ENABLED) != null) {
+ if (metadata != null &&
metadata.properties().get(TableProperties.HIVE_LOCK_ENABLED) != null) {
// We know that the property is set, so default value will not be used,
return metadata.propertyAsBoolean(TableProperties.HIVE_LOCK_ENABLED,
false);
}
@@ -596,19 +523,4 @@ public class HiveTableOperations extends
BaseMetastoreTableOperations {
return new NoLock();
}
}
-
- /**
- * Checks if the storage_handler property is already set to
HIVE_ICEBERG_STORAGE_HANDLER.
- * @param storageHandler Storage Handler class
- * @return true if the storage_handler property is set to
HIVE_ICEBERG_STORAGE_HANDLER
- */
- public static boolean isHiveIcebergStorageHandler(String storageHandler) {
- try {
- Class<?> storageHandlerClass = Class.forName(storageHandler);
- Class<?> icebergStorageHandlerClass =
Class.forName(HIVE_ICEBERG_STORAGE_HANDLER);
- return icebergStorageHandlerClass.isAssignableFrom(storageHandlerClass);
- } catch (ClassNotFoundException e) {
- throw new RuntimeException("Error checking storage handler class", e);
- }
- }
}
diff --git
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java
index ee62b64ed23..dd9fc44a5c8 100644
---
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java
+++
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.iceberg.common.DynMethods;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.thrift.TException;
public class MetastoreUtil {
private static final DynMethods.UnboundMethod ALTER_TABLE =
@@ -56,7 +57,7 @@ public class MetastoreUtil {
* context will be set in a way that turns off stats updates to avoid
recursive file listing.
*/
public static void alterTable(
- IMetaStoreClient client, String databaseName, String tblName, Table
table) {
+ IMetaStoreClient client, String databaseName, String tblName, Table
table) throws TException {
alterTable(client, databaseName, tblName, table, ImmutableMap.of());
}
@@ -69,11 +70,21 @@ public class MetastoreUtil {
String databaseName,
String tblName,
Table table,
- Map<String, String> extraEnv) {
+ Map<String, String> extraEnv)
+ throws TException {
Map<String, String> env = Maps.newHashMapWithExpectedSize(extraEnv.size()
+ 1);
env.putAll(extraEnv);
env.put(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE);
- ALTER_TABLE.invoke(client, databaseName, tblName, table, new
EnvironmentContext(env));
+ try {
+ ALTER_TABLE.invoke(client, databaseName, tblName, table, new
EnvironmentContext(env));
+ } catch (RuntimeException e) {
+ // TException would be wrapped into RuntimeException during reflection
+ if (e.getCause() instanceof TException) {
+ throw (TException) e.getCause();
+ } else {
+ throw e;
+ }
+ }
}
}
diff --git
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/HiveCreateReplaceTableTest.java
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/HiveCreateReplaceTableTest.java
index aa1790fd215..2c5246e8f9d 100644
---
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/HiveCreateReplaceTableTest.java
+++
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/HiveCreateReplaceTableTest.java
@@ -20,7 +20,10 @@
package org.apache.iceberg.hive;
import java.nio.file.Path;
+import java.util.concurrent.TimeUnit;
import org.apache.iceberg.AppendFiles;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.DataFiles;
import org.apache.iceberg.PartitionSpec;
@@ -36,8 +39,10 @@ import
org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.types.Types;
import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
@@ -47,8 +52,9 @@ import static
org.apache.iceberg.types.Types.NestedField.required;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
-public class HiveCreateReplaceTableTest extends HiveMetastoreTest {
+public class HiveCreateReplaceTableTest {
+ private static final String DB_NAME = "hivedb";
private static final String TABLE_NAME = "tbl";
private static final TableIdentifier TABLE_IDENTIFIER =
TableIdentifier.of(DB_NAME, TABLE_NAME);
private static final Schema SCHEMA = new Schema(
@@ -64,6 +70,25 @@ public class HiveCreateReplaceTableTest extends
HiveMetastoreTest {
private String tableLocation;
+ @RegisterExtension
+ private static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION =
+ HiveMetastoreExtension.builder().withDatabase(DB_NAME).build();
+
+ private static HiveCatalog catalog;
+
+ @BeforeAll
+ public static void initCatalog() {
+ catalog =
+ (HiveCatalog)
+ CatalogUtil.loadCatalog(
+ HiveCatalog.class.getName(),
+ CatalogUtil.ICEBERG_CATALOG_TYPE_HIVE,
+ ImmutableMap.of(
+ CatalogProperties.CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS,
+ String.valueOf(TimeUnit.SECONDS.toMillis(10))),
+ HIVE_METASTORE_EXTENSION.hiveConf());
+ }
+
@BeforeEach
public void createTableLocation() {
tableLocation = temp.resolve("hive-").toString();
diff --git
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/HiveMetastoreExtension.java
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/HiveMetastoreExtension.java
new file mode 100644
index 00000000000..238b8e50835
--- /dev/null
+++
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/HiveMetastoreExtension.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.hive;
+
+import java.util.Map;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.junit.jupiter.api.extension.AfterAllCallback;
+import org.junit.jupiter.api.extension.BeforeAllCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+
+public class HiveMetastoreExtension implements BeforeAllCallback,
AfterAllCallback {
+ private HiveMetaStoreClient metastoreClient;
+ private TestHiveMetastore metastore;
+ private final Map<String, String> hiveConfOverride;
+ private final String databaseName;
+
+ private HiveMetastoreExtension(String databaseName, Map<String, String>
hiveConfOverride) {
+ this.databaseName = databaseName;
+ this.hiveConfOverride = hiveConfOverride;
+ }
+
+ @Override
+ public void beforeAll(ExtensionContext extensionContext) throws Exception {
+ metastore = new TestHiveMetastore();
+ HiveConf hiveConfWithOverrides = new HiveConf(TestHiveMetastore.class);
+ if (hiveConfOverride != null) {
+ for (Map.Entry<String, String> kv : hiveConfOverride.entrySet()) {
+ hiveConfWithOverrides.set(kv.getKey(), kv.getValue());
+ }
+ }
+
+ metastore.start(hiveConfWithOverrides);
+ metastoreClient = new HiveMetaStoreClient(hiveConfWithOverrides);
+ if (null != databaseName) {
+ String dbPath = metastore.getDatabasePath(databaseName);
+ Database db = new Database(databaseName, "description", dbPath,
Maps.newHashMap());
+ metastoreClient.createDatabase(db);
+ }
+ }
+
+ @Override
+ public void afterAll(ExtensionContext extensionContext) throws Exception {
+ if (null != metastoreClient) {
+ metastoreClient.close();
+ }
+
+ if (null != metastore) {
+ metastore.stop();
+ }
+
+ metastoreClient = null;
+ metastore = null;
+ }
+
+ public HiveMetaStoreClient metastoreClient() {
+ return metastoreClient;
+ }
+
+ public HiveConf hiveConf() {
+ return metastore.hiveConf();
+ }
+
+ public TestHiveMetastore metastore() {
+ return metastore;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static class Builder {
+ private String databaseName;
+ private Map<String, String> config;
+
+ public Builder() {
+ }
+
+ public Builder withDatabase(String databaseToCreate) {
+ this.databaseName = databaseToCreate;
+ return this;
+ }
+
+ public Builder withConfig(Map<String, String> configToSet) {
+ this.config = configToSet;
+ return this;
+ }
+
+ public HiveMetastoreExtension build() {
+ return new HiveMetastoreExtension(databaseName, config);
+ }
+ }
+}
diff --git
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/HiveMetastoreTest.java
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/HiveMetastoreTest.java
deleted file mode 100644
index 27fbb41f7a0..00000000000
---
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/HiveMetastoreTest.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iceberg.hive;
-
-import java.util.Collections;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.iceberg.CatalogProperties;
-import org.apache.iceberg.CatalogUtil;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
-import org.apache.iceberg.relocated.com.google.common.collect.Maps;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.BeforeAll;
-
-public abstract class HiveMetastoreTest {
-
- protected static final String DB_NAME = "hivedb";
- protected static final long EVICTION_INTERVAL =
TimeUnit.SECONDS.toMillis(10);
-
- protected static HiveMetaStoreClient metastoreClient;
- protected static HiveCatalog catalog;
- protected static HiveConf hiveConf;
- protected static TestHiveMetastore metastore;
-
- @BeforeAll
- public static void startMetastore() throws Exception {
- startMetastore(Collections.emptyMap());
- }
-
- public static void startMetastore(Map<String, String> hiveConfOverride)
throws Exception {
- HiveMetastoreTest.metastore = new TestHiveMetastore();
- HiveConf hiveConfWithOverrides = new HiveConf(TestHiveMetastore.class);
- if (hiveConfOverride != null) {
- for (Map.Entry<String, String> kv : hiveConfOverride.entrySet()) {
- hiveConfWithOverrides.set(kv.getKey(), kv.getValue());
- }
- }
-
- metastore.start(hiveConfWithOverrides);
- HiveMetastoreTest.hiveConf = metastore.hiveConf();
- HiveMetastoreTest.metastoreClient = new
HiveMetaStoreClient(hiveConfWithOverrides);
- String dbPath = metastore.getDatabasePath(DB_NAME);
- Database db = new Database(DB_NAME, "description", dbPath,
Maps.newHashMap());
- metastoreClient.createDatabase(db);
- HiveMetastoreTest.catalog =
- (HiveCatalog)
- CatalogUtil.loadCatalog(
- HiveCatalog.class.getName(),
- CatalogUtil.ICEBERG_CATALOG_TYPE_HIVE,
- ImmutableMap.of(
- CatalogProperties.CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS,
- String.valueOf(EVICTION_INTERVAL)),
- hiveConfWithOverrides);
- }
-
- @AfterAll
- public static void stopMetastore() throws Exception {
- HiveMetastoreTest.catalog = null;
-
- metastoreClient.close();
- HiveMetastoreTest.metastoreClient = null;
-
- metastore.stop();
- HiveMetastoreTest.metastore = null;
- }
-}
diff --git
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/HiveTableBaseTest.java
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/HiveTableBaseTest.java
index ad6d78cdc91..321c1e59716 100644
---
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/HiveTableBaseTest.java
+++
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/HiveTableBaseTest.java
@@ -23,15 +23,21 @@ import java.io.File;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.apache.hadoop.fs.Path;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Schema;
import org.apache.iceberg.TableMetadataParser;
import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.types.Types;
import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.extension.RegisterExtension;
import static org.apache.iceberg.PartitionSpec.builderFor;
import static org.apache.iceberg.TableMetadataParser.getFileExtension;
@@ -39,11 +45,18 @@ import static
org.apache.iceberg.types.Types.NestedField.optional;
import static org.apache.iceberg.types.Types.NestedField.required;
-public class HiveTableBaseTest extends HiveMetastoreTest {
+public class HiveTableBaseTest {
static final String TABLE_NAME = "tbl";
+ static final String DB_NAME = "hivedb";
static final TableIdentifier TABLE_IDENTIFIER = TableIdentifier.of(DB_NAME,
TABLE_NAME);
+ @RegisterExtension
+ protected static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION =
+ HiveMetastoreExtension.builder().withDatabase(DB_NAME).build();
+
+ protected static HiveCatalog catalog;
+
static final Schema schema = new Schema(Types.StructType.of(
required(1, "id", Types.LongType.get())).fields());
@@ -55,6 +68,19 @@ public class HiveTableBaseTest extends HiveMetastoreTest {
private Path tableLocation;
+ @BeforeAll
+ public static void initCatalog() {
+ catalog =
+ (HiveCatalog)
+ CatalogUtil.loadCatalog(
+ HiveCatalog.class.getName(),
+ CatalogUtil.ICEBERG_CATALOG_TYPE_HIVE,
+ ImmutableMap.of(
+ CatalogProperties.CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS,
+ String.valueOf(TimeUnit.SECONDS.toMillis(10))),
+ HIVE_METASTORE_EXTENSION.hiveConf());
+ }
+
@BeforeEach
public void createTestTable() {
this.tableLocation = new Path(catalog.createTable(TABLE_IDENTIFIER,
schema, partitionSpec).location());
@@ -63,11 +89,11 @@ public class HiveTableBaseTest extends HiveMetastoreTest {
@AfterEach
public void dropTestTable() throws Exception {
// drop the table data
- tableLocation.getFileSystem(hiveConf).delete(tableLocation, true);
+
tableLocation.getFileSystem(HIVE_METASTORE_EXTENSION.hiveConf()).delete(tableLocation,
true);
catalog.dropTable(TABLE_IDENTIFIER, false /* metadata only, location was
already deleted */);
}
private static String getTableBasePath(String tableName) {
- String databasePath = metastore.getDatabasePath(DB_NAME);
+ String databasePath =
HIVE_METASTORE_EXTENSION.metastore().getDatabasePath(DB_NAME);
return Paths.get(databasePath, tableName).toAbsolutePath().toString();
}
diff --git
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/HiveTableTest.java
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/HiveTableTest.java
index a6207f3159a..a1e8255aefb 100644
---
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/HiveTableTest.java
+++
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/HiveTableTest.java
@@ -93,7 +93,9 @@ public class HiveTableTest extends HiveTableBaseTest {
// Table should be renamed in hive metastore
String tableName = TABLE_IDENTIFIER.name();
org.apache.hadoop.hive.metastore.api.Table table =
- metastoreClient.getTable(TABLE_IDENTIFIER.namespace().level(0),
tableName);
+ HIVE_METASTORE_EXTENSION
+ .metastoreClient()
+ .getTable(TABLE_IDENTIFIER.namespace().level(0), tableName);
// check parameters are in expected state
Map<String, String> parameters = table.getParameters();
@@ -272,7 +274,8 @@ public class HiveTableTest extends HiveTableBaseTest {
assertThat(manifestFiles(TABLE_NAME)).hasSize(0);
assertThat(icebergTable.schema().asStruct()).isEqualTo(altered.asStruct());
- final org.apache.hadoop.hive.metastore.api.Table table =
metastoreClient.getTable(DB_NAME, TABLE_NAME);
+ final org.apache.hadoop.hive.metastore.api.Table table =
+ HIVE_METASTORE_EXTENSION.metastoreClient().getTable(DB_NAME,
TABLE_NAME);
final List<String> hiveColumns = table.getSd().getCols().stream()
.map(FieldSchema::getName)
.collect(Collectors.toList());
@@ -318,10 +321,11 @@ public class HiveTableTest extends HiveTableBaseTest {
@Test
public void testFailure() throws TException {
Table icebergTable = catalog.loadTable(TABLE_IDENTIFIER);
- org.apache.hadoop.hive.metastore.api.Table table =
metastoreClient.getTable(DB_NAME, TABLE_NAME);
+ org.apache.hadoop.hive.metastore.api.Table table =
+ HIVE_METASTORE_EXTENSION.metastoreClient().getTable(DB_NAME,
TABLE_NAME);
String dummyLocation = "dummylocation";
table.getParameters().put(METADATA_LOCATION_PROP, dummyLocation);
- metastoreClient.alter_table(DB_NAME, TABLE_NAME, table);
+ HIVE_METASTORE_EXTENSION.metastoreClient().alter_table(DB_NAME,
TABLE_NAME, table);
assertThatThrownBy(() -> icebergTable.updateSchema()
.addColumn("data", Types.LongType.get())
.commit())
@@ -342,7 +346,7 @@ public class HiveTableTest extends HiveTableBaseTest {
// create a hive table
String hiveTableName = "test_hive_table";
org.apache.hadoop.hive.metastore.api.Table hiveTable =
createHiveTable(hiveTableName);
- metastoreClient.createTable(hiveTable);
+ HIVE_METASTORE_EXTENSION.metastoreClient().createTable(hiveTable);
catalog.setListAllTables(false);
List<TableIdentifier> tableIdents1 =
catalog.listTables(TABLE_IDENTIFIER.namespace());
@@ -353,7 +357,7 @@ public class HiveTableTest extends HiveTableBaseTest {
assertThat(tableIdents2).as("should be 2 tables in namespace
.").hasSize(2);
assertThat(catalog.tableExists(TABLE_IDENTIFIER)).isTrue();
- metastoreClient.dropTable(DB_NAME, hiveTableName);
+ HIVE_METASTORE_EXTENSION.metastoreClient().dropTable(DB_NAME,
hiveTableName);
}
private org.apache.hadoop.hive.metastore.api.Table createHiveTable(String
hiveTableName) throws IOException {
@@ -394,12 +398,13 @@ public class HiveTableTest extends HiveTableBaseTest {
assertThat(table.location()).isEqualTo(namespaceMeta.get("location") + "/"
+ TABLE_NAME);
// Drop the database and purge the files
- metastoreClient.dropDatabase(NON_DEFAULT_DATABASE, true, true, true);
+
HIVE_METASTORE_EXTENSION.metastoreClient().dropDatabase(NON_DEFAULT_DATABASE,
true, true, true);
}
@Test
public void testRegisterTable() throws TException {
- org.apache.hadoop.hive.metastore.api.Table originalTable =
metastoreClient.getTable(DB_NAME, TABLE_NAME);
+ org.apache.hadoop.hive.metastore.api.Table originalTable =
+ HIVE_METASTORE_EXTENSION.metastoreClient().getTable(DB_NAME,
TABLE_NAME);
Map<String, String> originalParams = originalTable.getParameters();
assertThat(originalParams).isNotNull();
@@ -414,7 +419,8 @@ public class HiveTableTest extends HiveTableBaseTest {
catalog.registerTable(TABLE_IDENTIFIER, "file:" +
metadataVersionFiles.get(0));
- org.apache.hadoop.hive.metastore.api.Table newTable =
metastoreClient.getTable(DB_NAME, TABLE_NAME);
+ org.apache.hadoop.hive.metastore.api.Table newTable =
+ HIVE_METASTORE_EXTENSION.metastoreClient().getTable(DB_NAME,
TABLE_NAME);
Map<String, String> newTableParameters = newTable.getParameters();
assertThat(newTableParameters)
@@ -448,7 +454,7 @@ public class HiveTableTest extends HiveTableBaseTest {
.collect(Collectors.toList());
assertThat(metadataFiles).hasSize(2);
- assertThatThrownBy(() -> metastoreClient.getTable(DB_NAME, "table1"))
+ assertThatThrownBy(() ->
HIVE_METASTORE_EXTENSION.metastoreClient().getTable(DB_NAME, "table1"))
.isInstanceOf(NoSuchObjectException.class)
.hasMessage("hive.hivedb.table1 table not found");
assertThatThrownBy(() -> catalog.loadTable(identifier))
@@ -458,7 +464,7 @@ public class HiveTableTest extends HiveTableBaseTest {
// register the table to hive catalog using the latest metadata file
String latestMetadataFile = ((BaseTable)
table).operations().current().metadataFileLocation();
catalog.registerTable(identifier, "file:" + latestMetadataFile);
- assertThat(metastoreClient.getTable(DB_NAME, "table1")).isNotNull();
+ assertThat(HIVE_METASTORE_EXTENSION.metastoreClient().getTable(DB_NAME,
"table1")).isNotNull();
// load the table in hive catalog
table = catalog.loadTable(identifier);
@@ -504,7 +510,8 @@ public class HiveTableTest extends HiveTableBaseTest {
@Test
public void testRegisterExistingTable() throws TException {
- org.apache.hadoop.hive.metastore.api.Table originalTable =
metastoreClient.getTable(DB_NAME, TABLE_NAME);
+ org.apache.hadoop.hive.metastore.api.Table originalTable =
+ HIVE_METASTORE_EXTENSION.metastoreClient().getTable(DB_NAME,
TABLE_NAME);
Map<String, String> originalParams = originalTable.getParameters();
assertThat(originalParams).isNotNull();
@@ -530,7 +537,8 @@ public class HiveTableTest extends HiveTableBaseTest {
catalog.getConf().setBoolean(ConfigProperties.ENGINE_HIVE_ENABLED, false);
catalog.createTable(TABLE_IDENTIFIER, schema,
PartitionSpec.unpartitioned());
- org.apache.hadoop.hive.metastore.api.Table hmsTable =
metastoreClient.getTable(DB_NAME, TABLE_NAME);
+ org.apache.hadoop.hive.metastore.api.Table hmsTable =
+ HIVE_METASTORE_EXTENSION.metastoreClient().getTable(DB_NAME,
TABLE_NAME);
assertHiveEnabled(hmsTable, false);
}
@@ -543,7 +551,8 @@ public class HiveTableTest extends HiveTableBaseTest {
// Enable by hive-conf
catalog.createTable(TABLE_IDENTIFIER, schema,
PartitionSpec.unpartitioned());
- org.apache.hadoop.hive.metastore.api.Table hmsTable =
metastoreClient.getTable(DB_NAME, TABLE_NAME);
+ org.apache.hadoop.hive.metastore.api.Table hmsTable =
+ HIVE_METASTORE_EXTENSION.metastoreClient().getTable(DB_NAME,
TABLE_NAME);
assertHiveEnabled(hmsTable, true);
@@ -553,7 +562,7 @@ public class HiveTableTest extends HiveTableBaseTest {
catalog.getConf().setBoolean(ConfigProperties.ENGINE_HIVE_ENABLED, false);
catalog.createTable(TABLE_IDENTIFIER, schema,
PartitionSpec.unpartitioned());
- hmsTable = metastoreClient.getTable(DB_NAME, TABLE_NAME);
+ hmsTable = HIVE_METASTORE_EXTENSION.metastoreClient().getTable(DB_NAME,
TABLE_NAME);
assertHiveEnabled(hmsTable, false);
}
@@ -569,7 +578,8 @@ public class HiveTableTest extends HiveTableBaseTest {
catalog.getConf().set(ConfigProperties.ENGINE_HIVE_ENABLED, "false");
catalog.createTable(TABLE_IDENTIFIER, schema,
PartitionSpec.unpartitioned(), tableProperties);
- org.apache.hadoop.hive.metastore.api.Table hmsTable =
metastoreClient.getTable(DB_NAME, TABLE_NAME);
+ org.apache.hadoop.hive.metastore.api.Table hmsTable =
+ HIVE_METASTORE_EXTENSION.metastoreClient().getTable(DB_NAME,
TABLE_NAME);
assertHiveEnabled(hmsTable, true);
@@ -580,7 +590,7 @@ public class HiveTableTest extends HiveTableBaseTest {
catalog.getConf().set(ConfigProperties.ENGINE_HIVE_ENABLED, "true");
catalog.createTable(TABLE_IDENTIFIER, schema,
PartitionSpec.unpartitioned(), tableProperties);
- hmsTable = metastoreClient.getTable(DB_NAME, TABLE_NAME);
+ hmsTable = HIVE_METASTORE_EXTENSION.metastoreClient().getTable(DB_NAME,
TABLE_NAME);
assertHiveEnabled(hmsTable, false);
}
diff --git
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestCachedClientPool.java
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestCachedClientPool.java
index 8348c8f3293..f4e3c25a6d7 100644
---
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestCachedClientPool.java
+++
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestCachedClientPool.java
@@ -20,39 +20,53 @@
package org.apache.iceberg.hive;
import java.security.PrivilegedAction;
-import java.util.Collections;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.CatalogUtil;
import org.apache.iceberg.exceptions.ValidationException;
import org.apache.iceberg.hive.CachedClientPool.Key;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_HIVE;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
-public class TestCachedClientPool extends HiveMetastoreTest {
+public class TestCachedClientPool {
+ private static final long EVICTION_INTERVAL = TimeUnit.SECONDS.toMillis(10);
+ private static final String DB_NAME = "hivedb";
+
+ @RegisterExtension
+ private static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION =
+ HiveMetastoreExtension.builder().withDatabase(DB_NAME).build();
@Test
public void testClientPoolCleaner() throws InterruptedException {
- CachedClientPool clientPool = new CachedClientPool(hiveConf,
Collections.emptyMap());
+ CachedClientPool clientPool =
+ new CachedClientPool(
+ HIVE_METASTORE_EXTENSION.hiveConf(),
+ ImmutableMap.of(
+ CatalogProperties.CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS,
+ String.valueOf(EVICTION_INTERVAL)));
HiveClientPool clientPool1 = clientPool.clientPool();
assertThat(clientPool1)
.isSameAs(
CachedClientPool.clientPoolCache()
- .getIfPresent(CachedClientPool.extractKey(null, hiveConf)));
+ .getIfPresent(
+ CachedClientPool.extractKey(null,
HIVE_METASTORE_EXTENSION.hiveConf())));
TimeUnit.MILLISECONDS.sleep(EVICTION_INTERVAL -
TimeUnit.SECONDS.toMillis(2));
HiveClientPool clientPool2 = clientPool.clientPool();
assertThat(clientPool2).isSameAs(clientPool1);
TimeUnit.MILLISECONDS.sleep(EVICTION_INTERVAL +
TimeUnit.SECONDS.toMillis(5));
assertThat(
CachedClientPool.clientPoolCache()
- .getIfPresent(CachedClientPool.extractKey(null, hiveConf)))
+ .getIfPresent(CachedClientPool.extractKey(null,
HIVE_METASTORE_EXTENSION.hiveConf())))
.isNull();
// The client has been really closed.
@@ -67,6 +81,8 @@ public class TestCachedClientPool extends HiveMetastoreTest {
UserGroupInformation foo2 = UserGroupInformation.createProxyUser("foo",
current);
UserGroupInformation bar = UserGroupInformation.createProxyUser("bar",
current);
+ HiveConf hiveConf = HIVE_METASTORE_EXTENSION.hiveConf();
+
Key key1 =
foo1.doAs(
(PrivilegedAction<Key>)
diff --git
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveCatalog.java
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveCatalog.java
index aa55094f12b..3aef0bd8fe5 100644
---
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveCatalog.java
+++
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveCatalog.java
@@ -21,10 +21,10 @@ package org.apache.iceberg.hive;
import java.io.IOException;
import java.nio.file.Path;
-import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.Database;
@@ -36,7 +36,6 @@ import org.apache.iceberg.CatalogUtil;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.DataFiles;
import org.apache.iceberg.FileFormat;
-import org.apache.iceberg.HasTableOperations;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.PartitionSpecParser;
import org.apache.iceberg.Schema;
@@ -48,10 +47,10 @@ import org.apache.iceberg.Table;
import org.apache.iceberg.TableMetadata;
import org.apache.iceberg.TableOperations;
import org.apache.iceberg.TableProperties;
-import org.apache.iceberg.TestHelpers;
import org.apache.iceberg.Transaction;
import org.apache.iceberg.UpdateSchema;
import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.CatalogTests;
import org.apache.iceberg.catalog.Namespace;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.exceptions.AlreadyExistsException;
@@ -66,7 +65,11 @@ import org.apache.iceberg.transforms.Transforms;
import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.JsonUtil;
import org.apache.thrift.TException;
+import org.junit.Assert;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
@@ -88,7 +91,10 @@ import static
org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-public class TestHiveCatalog extends HiveMetastoreTest {
+/**
+ * Run all the tests from abstract of {@link CatalogTests} with few specific
tests related to HIVE.
+ */
+public class TestHiveCatalog extends CatalogTests<HiveCatalog> {
private static ImmutableMap meta = ImmutableMap.of(
"owner", "apache",
"group", "iceberg",
@@ -97,6 +103,54 @@ public class TestHiveCatalog extends HiveMetastoreTest {
@TempDir
private Path temp;
+ private HiveCatalog catalog;
+ private static final String DB_NAME = "hivedb";
+
+ @RegisterExtension
+ private static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION =
+ HiveMetastoreExtension.builder().build();
+
+ @BeforeEach
+ public void before() throws TException {
+ catalog =
+ (HiveCatalog)
+ CatalogUtil.loadCatalog(
+ HiveCatalog.class.getName(),
+ CatalogUtil.ICEBERG_CATALOG_TYPE_HIVE,
+ ImmutableMap.of(
+ CatalogProperties.CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS,
+ String.valueOf(TimeUnit.SECONDS.toMillis(10))),
+ HIVE_METASTORE_EXTENSION.hiveConf());
+ String dbPath =
HIVE_METASTORE_EXTENSION.metastore().getDatabasePath(DB_NAME);
+ Database db = new Database(DB_NAME, "description", dbPath,
Maps.newHashMap());
+ HIVE_METASTORE_EXTENSION.metastoreClient().createDatabase(db);
+ }
+
+ @AfterEach
+ public void cleanup() throws Exception {
+ HIVE_METASTORE_EXTENSION.metastore().reset();
+ }
+
+ @Override
+ protected boolean requiresNamespaceCreate() {
+ return true;
+ }
+
+ @Override
+ protected boolean supportsNamesWithSlashes() {
+ return false;
+ }
+
+ @Override
+ protected boolean supportsNamesWithDot() {
+ return false;
+ }
+
+ @Override
+ protected HiveCatalog catalog() {
+ return catalog;
+ }
+
private Schema getTestSchema() {
return new Schema(
required(1, "id", Types.IntegerType.get(), "unique ID"),
@@ -165,8 +219,8 @@ public class TestHiveCatalog extends HiveMetastoreTest {
assertThatNoException()
.isThrownBy(
() -> {
- HiveCatalog catalog = new HiveCatalog();
- catalog.initialize("hive", Maps.newHashMap());
+ HiveCatalog hiveCatalog = new HiveCatalog();
+ hiveCatalog.initialize("hive", Maps.newHashMap());
});
}
@@ -175,8 +229,8 @@ public class TestHiveCatalog extends HiveMetastoreTest {
assertThatNoException()
.isThrownBy(
() -> {
- HiveCatalog catalog = new HiveCatalog();
- catalog.toString();
+ HiveCatalog hiveCatalog = new HiveCatalog();
+ hiveCatalog.toString();
});
}
@@ -185,11 +239,12 @@ public class TestHiveCatalog extends HiveMetastoreTest {
Map<String, String> properties = Maps.newHashMap();
properties.put("uri", "thrift://examplehost:9083");
properties.put("warehouse", "/user/hive/testwarehouse");
- HiveCatalog catalog = new HiveCatalog();
- catalog.initialize("hive", properties);
+ HiveCatalog hiveCatalog = new HiveCatalog();
+ hiveCatalog.initialize("hive", properties);
-
assertThat(catalog.getConf().get("hive.metastore.uris")).isEqualTo("thrift://examplehost:9083");
- assertThat(catalog.getConf().get("hive.metastore.warehouse.dir"))
+ assertThat(hiveCatalog.getConf().get("hive.metastore.uris"))
+ .isEqualTo("thrift://examplehost:9083");
+ assertThat(hiveCatalog.getConf().get("hive.metastore.warehouse.dir"))
.isEqualTo("/user/hive/testwarehouse");
}
@@ -288,7 +343,8 @@ public class TestHiveCatalog extends HiveMetastoreTest {
String location = temp.resolve(tbl).toString();
try {
Table table = catalog.createTable(tableIdent, schema, spec, location,
properties);
- org.apache.hadoop.hive.metastore.api.Table hmsTable =
metastoreClient.getTable(db, tbl);
+ org.apache.hadoop.hive.metastore.api.Table hmsTable =
+ HIVE_METASTORE_EXTENSION.metastoreClient().getTable(db, tbl);
assertThat(hmsTable.getOwner()).isEqualTo(owner);
Map<String, String> hmsTableParams = hmsTable.getParameters();
assertThat(hmsTableParams).doesNotContainKey(HiveCatalog.HMS_TABLE_OWNER);
@@ -348,10 +404,11 @@ public class TestHiveCatalog extends HiveMetastoreTest {
}
@Test
- public void testCreateNamespace() throws Exception {
+ public void testDatabaseAndNamespaceWithLocation() throws Exception {
Namespace namespace1 = Namespace.of("noLocation");
catalog.createNamespace(namespace1, meta);
- Database database1 = metastoreClient.getDatabase(namespace1.toString());
+ Database database1 =
+
HIVE_METASTORE_EXTENSION.metastoreClient().getDatabase(namespace1.toString());
assertThat(database1.getParameters()).containsEntry("owner", "apache");
assertThat(database1.getParameters()).containsEntry("group", "iceberg");
@@ -362,7 +419,7 @@ public class TestHiveCatalog extends HiveMetastoreTest {
assertThatThrownBy(() -> catalog.createNamespace(namespace1))
.isInstanceOf(AlreadyExistsException.class)
- .hasMessage("Namespace '" + namespace1 + "' already exists!");
+ .hasMessage(String.format("Namespace already exists: %s", namespace1));
String hiveLocalDir = temp.toFile().toURI().toString();
// remove the trailing slash of the URI
hiveLocalDir = hiveLocalDir.substring(0, hiveLocalDir.length() - 1);
@@ -373,7 +430,8 @@ public class TestHiveCatalog extends HiveMetastoreTest {
Namespace namespace2 = Namespace.of("haveLocation");
catalog.createNamespace(namespace2, newMeta);
- Database database2 = metastoreClient.getDatabase(namespace2.toString());
+ Database database2 =
+
HIVE_METASTORE_EXTENSION.metastoreClient().getDatabase(namespace2.toString());
assertThat(hiveLocalDir)
.as("There no same location for db and namespace")
.isEqualTo(database2.getLocationUri());
@@ -453,27 +511,12 @@ public class TestHiveCatalog extends HiveMetastoreTest {
Namespace namespace = Namespace.of(name);
catalog.createNamespace(namespace, prop);
- Database db = metastoreClient.getDatabase(namespace.toString());
+ Database db =
HIVE_METASTORE_EXTENSION.metastoreClient().getDatabase(namespace.toString());
assertThat(db.getOwnerName()).isEqualTo(expectedOwner);
assertThat(db.getOwnerType()).isEqualTo(expectedOwnerType);
}
- @Test
- public void testListNamespace() throws TException {
- List<Namespace> namespaces;
- Namespace namespace1 = Namespace.of("dbname1");
- catalog.createNamespace(namespace1, meta);
- namespaces = catalog.listNamespaces(namespace1);
- assertThat(namespaces).as("Hive db not hive the namespace
'dbname1'").isEmpty();
-
- Namespace namespace2 = Namespace.of("dbname2");
- catalog.createNamespace(namespace2, meta);
- namespaces = catalog.listNamespaces();
-
- assertThat(namespaces).as("Hive db not hive the namespace
'dbname2'").contains(namespace2);
- }
-
@Test
public void testLoadNamespaceMeta() throws TException {
Namespace namespace = Namespace.of("dbname_load");
@@ -489,7 +532,7 @@ public class TestHiveCatalog extends HiveMetastoreTest {
}
@Test
- public void testNamespaceExists() throws TException {
+ public void testNamespaceExists() {
Namespace namespace = Namespace.of("dbname_exists");
catalog.createNamespace(namespace, meta);
@@ -501,7 +544,7 @@ public class TestHiveCatalog extends HiveMetastoreTest {
}
@Test
- public void testSetNamespaceProperties() throws TException {
+ public void testSetNamespaceProperties() {
Namespace namespace = Namespace.of("dbname_set");
catalog.createNamespace(namespace, meta);
@@ -513,7 +556,12 @@ public class TestHiveCatalog extends HiveMetastoreTest {
"comment", "iceberg test")
);
- Database database = metastoreClient.getDatabase(namespace.level(0));
+ Database database = null;
+ try {
+ database =
HIVE_METASTORE_EXTENSION.metastoreClient().getDatabase(namespace.level(0));
+ } catch (TException e) {
+ Assert.fail();
+ }
assertThat(database.getParameters()).containsEntry("owner",
"alter_apache");
assertThat(database.getParameters()).containsEntry("test", "test");
assertThat(database.getParameters()).containsEntry("group", "iceberg");
@@ -698,22 +746,26 @@ public class TestHiveCatalog extends HiveMetastoreTest {
name, propToCreate, expectedOwnerPostCreate,
expectedOwnerTypePostCreate);
catalog.setProperties(Namespace.of(name), propToSet);
- Database database = metastoreClient.getDatabase(name);
+ Database database =
HIVE_METASTORE_EXTENSION.metastoreClient().getDatabase(name);
assertThat(database.getOwnerName()).isEqualTo(expectedOwnerPostSet);
assertThat(database.getOwnerType()).isEqualTo(expectedOwnerTypePostSet);
}
@Test
- public void testRemoveNamespaceProperties() throws TException {
+ public void testRemoveNamespaceProperties() {
Namespace namespace = Namespace.of("dbname_remove");
catalog.createNamespace(namespace, meta);
catalog.removeProperties(namespace, ImmutableSet.of("comment", "owner"));
- Database database = metastoreClient.getDatabase(namespace.level(0));
-
+ Database database = null;
+ try {
+ database =
HIVE_METASTORE_EXTENSION.metastoreClient().getDatabase(namespace.level(0));
+ } catch (TException e) {
+ Assert.fail();
+ }
assertThat(database.getParameters()).doesNotContainKey("owner");
assertThat(database.getParameters()).containsEntry("group", "iceberg");
@@ -844,14 +896,14 @@ public class TestHiveCatalog extends HiveMetastoreTest {
catalog.removeProperties(Namespace.of(name), propToRemove);
- Database database = metastoreClient.getDatabase(name);
+ Database database =
HIVE_METASTORE_EXTENSION.metastoreClient().getDatabase(name);
assertThat(database.getOwnerName()).isEqualTo(expectedOwnerPostRemove);
assertThat(database.getOwnerType()).isEqualTo(expectedOwnerTypePostRemove);
}
@Test
- public void testDropNamespace() throws TException {
+ public void dropNamespace() {
Namespace namespace = Namespace.of("dbname_drop");
TableIdentifier identifier = TableIdentifier.of(namespace, "table");
Schema schema = getTestSchema();
@@ -917,7 +969,7 @@ public class TestHiveCatalog extends HiveMetastoreTest {
}
private String defaultUri(Namespace namespace) throws TException {
- return metastoreClient.getConfigValue(
+ return HIVE_METASTORE_EXTENSION.metastoreClient().getConfigValue(
"hive.metastore.warehouse.external.dir", "") + "/" +
namespace.level(0) + ".db";
}
@@ -950,8 +1002,8 @@ public class TestHiveCatalog extends HiveMetastoreTest {
.create();
String tableName = tableIdentifier.name();
- org.apache.hadoop.hive.metastore.api.Table hmsTable =
- metastoreClient.getTable(tableIdentifier.namespace().level(0),
tableName);
+ org.apache.hadoop.hive.metastore.api.Table hmsTable =
HIVE_METASTORE_EXTENSION.metastoreClient().getTable(
+ tableIdentifier.namespace().level(0), tableName);
// check whether parameters are in expected state
Map<String, String> parameters = hmsTableParameters();
@@ -1095,7 +1147,8 @@ public class TestHiveCatalog extends HiveMetastoreTest {
}
private Map<String, String> hmsTableParameters() throws TException {
- org.apache.hadoop.hive.metastore.api.Table hmsTable =
metastoreClient.getTable(DB_NAME, "tbl");
+ org.apache.hadoop.hive.metastore.api.Table hmsTable =
+ HIVE_METASTORE_EXTENSION.metastoreClient().getTable(DB_NAME, "tbl");
return hmsTable.getParameters();
}
@@ -1128,7 +1181,7 @@ public class TestHiveCatalog extends HiveMetastoreTest {
HiveCatalog.class.getName(),
CatalogUtil.ICEBERG_CATALOG_TYPE_HIVE,
catalogProps,
- hiveConf);
+ HIVE_METASTORE_EXTENSION.hiveConf());
try {
Table table =
@@ -1168,42 +1221,11 @@ public class TestHiveCatalog extends HiveMetastoreTest {
conf.set("hive.metastore.warehouse.dir", "s3://bucket/");
conf.set("hive.metastore.warehouse.external.dir", "s3://bucket/");
- HiveCatalog catalog = new HiveCatalog();
- catalog.setConf(conf);
+ HiveCatalog hiveCatalog = new HiveCatalog();
+ hiveCatalog.setConf(conf);
- Database database = catalog.convertToDatabase(Namespace.of("database"),
ImmutableMap.of());
+ Database database =
hiveCatalog.convertToDatabase(Namespace.of("database"), ImmutableMap.of());
assertThat(database.getLocationUri()).isEqualTo("s3://bucket/database.db");
}
-
- @Test
- public void testRegisterTable() {
- TableIdentifier identifier = TableIdentifier.of(DB_NAME, "t1");
- catalog.createTable(identifier, getTestSchema());
- Table registeringTable = catalog.loadTable(identifier);
- catalog.dropTable(identifier, false);
- TableOperations ops = ((HasTableOperations) registeringTable).operations();
- String metadataLocation = ((HiveTableOperations)
ops).currentMetadataLocation();
- Table registeredTable = catalog.registerTable(identifier,
metadataLocation);
- assertThat(registeredTable).isNotNull();
- TestHelpers.assertSerializedAndLoadedMetadata(registeringTable,
registeredTable);
- String expectedMetadataLocation =
- ((HasTableOperations)
registeredTable).operations().current().metadataFileLocation();
- assertThat(metadataLocation).isEqualTo(expectedMetadataLocation);
- assertThat(catalog.loadTable(identifier)).isNotNull();
- assertThat(catalog.dropTable(identifier)).isTrue();
- }
-
- @Test
- public void testRegisterExistingTable() {
- TableIdentifier identifier = TableIdentifier.of(DB_NAME, "t1");
- catalog.createTable(identifier, getTestSchema());
- Table registeringTable = catalog.loadTable(identifier);
- TableOperations ops = ((HasTableOperations) registeringTable).operations();
- String metadataLocation = ((HiveTableOperations)
ops).currentMetadataLocation();
- assertThatThrownBy(() -> catalog.registerTable(identifier,
metadataLocation))
- .isInstanceOf(AlreadyExistsException.class)
- .hasMessage("Table already exists: hivedb.t1");
- assertThat(catalog.dropTable(identifier, true)).isTrue();
- }
}
diff --git
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveCommitLocks.java
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveCommitLocks.java
index a9095a89ce4..65ccfebd987 100644
---
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveCommitLocks.java
+++
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveCommitLocks.java
@@ -27,6 +27,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.IntStream;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
@@ -36,9 +37,14 @@ import org.apache.hadoop.hive.metastore.api.LockResponse;
import org.apache.hadoop.hive.metastore.api.LockState;
import org.apache.hadoop.hive.metastore.api.ShowLocksResponse;
import org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
import org.apache.iceberg.Table;
import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.exceptions.CommitFailedException;
import org.apache.iceberg.hadoop.ConfigProperties;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
@@ -46,14 +52,18 @@ import
org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.types.Types;
import org.apache.thrift.TException;
import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
import org.mockito.AdditionalAnswers;
import org.mockito.ArgumentCaptor;
import org.mockito.MockedStatic;
import org.mockito.invocation.InvocationOnMock;
+import static org.apache.iceberg.PartitionSpec.builderFor;
+import static org.apache.iceberg.types.Types.NestedField.required;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Mockito.any;
@@ -72,7 +82,7 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-public class TestHiveCommitLocks extends HiveTableBaseTest {
+public class TestHiveCommitLocks {
private static HiveTableOperations spyOps = null;
private static HiveClientPool spyClientPool = null;
private static CachedClientPool spyCachedClientPool = null;
@@ -89,13 +99,37 @@ public class TestHiveCommitLocks extends HiveTableBaseTest {
LockResponse notAcquiredLockResponse = new LockResponse(dummyLockId,
LockState.NOT_ACQUIRED);
ShowLocksResponse emptyLocks = new ShowLocksResponse(Lists.newArrayList());
+ private static final String DB_NAME = "hivedb";
+ private static final String TABLE_NAME = "tbl";
+ private static final Schema schema =
+ new Schema(Types.StructType.of(required(1, "id",
Types.LongType.get())).fields());
+ private static final PartitionSpec partitionSpec =
builderFor(schema).identity("id").build();
+ static final TableIdentifier TABLE_IDENTIFIER = TableIdentifier.of(DB_NAME,
TABLE_NAME);
+
+ @RegisterExtension
+ private static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION =
+ HiveMetastoreExtension.builder()
+ .withDatabase(DB_NAME)
+
.withConfig(ImmutableMap.of(HiveConf.ConfVars.HIVE_TXN_TIMEOUT.varname, "1s"))
+ .build();
+
+ private static HiveCatalog catalog;
+ private Path tableLocation;
+
@BeforeAll
- public static void startMetastore() throws Exception {
- HiveMetastoreTest.startMetastore(
- ImmutableMap.of(HiveConf.ConfVars.HIVE_TXN_TIMEOUT.varname, "1s"));
+ public static void initCatalog() throws Exception {
+ catalog =
+ (HiveCatalog)
+ CatalogUtil.loadCatalog(
+ HiveCatalog.class.getName(),
+ CatalogUtil.ICEBERG_CATALOG_TYPE_HIVE,
+ ImmutableMap.of(
+ CatalogProperties.CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS,
+ String.valueOf(TimeUnit.SECONDS.toMillis(10))),
+ HIVE_METASTORE_EXTENSION.hiveConf());
// start spies
- overriddenHiveConf = new Configuration(hiveConf);
+ overriddenHiveConf = new
Configuration(HIVE_METASTORE_EXTENSION.hiveConf());
overriddenHiveConf.setLong("iceberg.hive.lock-timeout-ms", 6 * 1000);
overriddenHiveConf.setLong("iceberg.hive.lock-check-min-wait-ms", 50);
overriddenHiveConf.setLong("iceberg.hive.lock-check-max-wait-ms", 5 *
1000);
@@ -106,14 +140,16 @@ public class TestHiveCommitLocks extends
HiveTableBaseTest {
spyClientPool = spy(new HiveClientPool(1, overriddenHiveConf));
when(spyClientPool.newClient()).thenAnswer(invocation -> {
// cannot spy on RetryingHiveMetastoreClient as it is a proxy
- IMetaStoreClient client = spy(new HiveMetaStoreClient(hiveConf));
+ IMetaStoreClient client =
+ spy(new HiveMetaStoreClient(HIVE_METASTORE_EXTENSION.hiveConf()));
spyClientRef.set(client);
return spyClientRef.get();
});
spyClientPool.run(IMetaStoreClient::isLocalMetaStore); // To ensure new
client is created.
- spyCachedClientPool = spy(new CachedClientPool(hiveConf,
Collections.emptyMap()));
+ spyCachedClientPool =
+ spy(new CachedClientPool(HIVE_METASTORE_EXTENSION.hiveConf(),
Collections.emptyMap()));
when(spyCachedClientPool.clientPool()).thenAnswer(invocation ->
spyClientPool);
assertThat(spyClientRef.get()).isNotNull();
@@ -123,6 +159,8 @@ public class TestHiveCommitLocks extends HiveTableBaseTest {
@BeforeEach
public void before() throws Exception {
+ this.tableLocation =
+ new Path(catalog.createTable(TABLE_IDENTIFIER, schema,
partitionSpec).location());
Table table = catalog.loadTable(TABLE_IDENTIFIER);
ops = (HiveTableOperations) ((HasTableOperations) table).operations();
String dbName = TABLE_IDENTIFIER.namespace().level(0);
@@ -152,6 +190,13 @@ public class TestHiveCommitLocks extends HiveTableBaseTest
{
reset(spyClient);
}
+ @AfterEach
+ public void dropTestTable() throws Exception {
+ // drop the table data
+
tableLocation.getFileSystem(HIVE_METASTORE_EXTENSION.hiveConf()).delete(tableLocation,
true);
+ catalog.dropTable(TABLE_IDENTIFIER, false /* metadata only, location was
already deleted */);
+ }
+
@AfterAll
public static void cleanup() {
try {
diff --git
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveCommits.java
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveCommits.java
index 97c1ed15e07..411ed6d3786 100644
---
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveCommits.java
+++
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveCommits.java
@@ -30,10 +30,12 @@ import org.apache.iceberg.exceptions.AlreadyExistsException;
import org.apache.iceberg.exceptions.CommitFailedException;
import org.apache.iceberg.exceptions.CommitStateUnknownException;
import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.types.Types;
import org.apache.thrift.TException;
import org.junit.jupiter.api.Test;
+import static org.apache.iceberg.TableProperties.HIVE_LOCK_ENABLED;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Matchers.any;
@@ -67,7 +69,7 @@ public class TestHiveCommits extends HiveTableBaseTest {
AtomicReference<HiveLock> lockRef = new AtomicReference<>();
- when(spyOps.lockObject(metadataV1))
+ when(spyOps.lockObject(metadataV2))
.thenAnswer(
i -> {
HiveLock lock = (HiveLock) i.callRealMethod();
@@ -283,11 +285,11 @@ public class TestHiveCommits extends HiveTableBaseTest {
AtomicReference<HiveLock> lock = new AtomicReference<>();
doAnswer(
l -> {
- lock.set(ops.lockObject(metadataV1));
+ lock.set(ops.lockObject(metadataV2));
return lock.get();
})
.when(spyOps)
- .lockObject(metadataV1);
+ .lockObject(metadataV2);
concurrentCommitAndThrowException(ops, spyOps, table, lock);
@@ -408,6 +410,55 @@ public class TestHiveCommits extends HiveTableBaseTest {
.isTrue();
}
+ @Test
+ public void testCommitExceptionWithoutMessage() throws TException,
InterruptedException {
+ Table table = catalog.loadTable(TABLE_IDENTIFIER);
+ HiveTableOperations ops = (HiveTableOperations) ((HasTableOperations)
table).operations();
+
+ TableMetadata metadataV1 = ops.current();
+ table.updateSchema().addColumn("n", Types.IntegerType.get()).commit();
+
+ ops.refresh();
+
+ HiveTableOperations spyOps = spy(ops);
+ doThrow(new RuntimeException()).when(spyOps).persistTable(any(),
anyBoolean(), any());
+
+ assertThatThrownBy(() -> spyOps.commit(ops.current(), metadataV1))
+ .isInstanceOf(CommitStateUnknownException.class)
+ .hasMessageStartingWith("null\nCannot determine whether the commit was
successful or not");
+ }
+
+ @Test
+ public void testChangeLockWithAlterTable() throws Exception {
+ Table table = catalog.loadTable(TABLE_IDENTIFIER);
+ HiveTableOperations ops = (HiveTableOperations) ((HasTableOperations)
table).operations();
+ TableMetadata base = ops.current();
+ final HiveLock initialLock = ops.lockObject(base);
+
+ AtomicReference<HiveLock> lockRef = new AtomicReference<>();
+ HiveTableOperations spyOps = spy(ops);
+ doAnswer(
+ i -> {
+ lockRef.set(ops.lockObject(i.getArgument(0)));
+ return lockRef.get();
+ })
+ .when(spyOps)
+ .lockObject(base);
+
+ TableMetadata newMetadata =
+ TableMetadata.buildFrom(base)
+ .setProperties(
+ ImmutableMap.of(
+ HIVE_LOCK_ENABLED, initialLock instanceof NoLock ? "true"
: "false"))
+ .build();
+ spyOps.commit(base, newMetadata);
+
+ assertThat(lockRef).as("Lock not captured by the
stub").doesNotHaveValue(null);
+ assertThat(lockRef.get())
+ .as("New lock mechanism shouldn't take effect before the commit
completes")
+ .hasSameClassAs(initialLock);
+ }
+
private void commitAndThrowException(
HiveTableOperations realOperations, HiveTableOperations spyOperations)
throws TException, InterruptedException {
diff --git
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java
index d839a0c65eb..1b2bae823ce 100644
---
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java
+++
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java
@@ -20,6 +20,7 @@
package org.apache.iceberg.hive;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -119,7 +120,8 @@ public class TestHiveSchemaUtil {
public void testNotSupportedTypes() {
for (FieldSchema notSupportedField : getNotSupportedFieldSchemas()) {
assertThatThrownBy(
- () ->
HiveSchemaUtil.convert(Lists.newArrayList(Arrays.asList(notSupportedField))))
+ () -> HiveSchemaUtil.convert(
+
Lists.newArrayList(Collections.singletonList(notSupportedField))))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageStartingWith("Unsupported Hive type");
}
@@ -164,7 +166,7 @@ public class TestHiveSchemaUtil {
Arrays.asList("customer_id", "first_name"),
Arrays.asList(TypeInfoUtils.getTypeInfoFromTypeString(serdeConstants.BIGINT_TYPE_NAME),
TypeInfoUtils.getTypeInfoFromTypeString(serdeConstants.STRING_TYPE_NAME)),
- Arrays.asList("customer comment"));
+ Collections.singletonList("customer comment"));
assertThat(schema.asStruct()).isEqualTo(expected.asStruct());
}
diff --git
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveTableConcurrency.java
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveTableConcurrency.java
index 06a10af4371..5b0fd740f00 100644
---
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveTableConcurrency.java
+++
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveTableConcurrency.java
@@ -19,6 +19,7 @@
package org.apache.iceberg.hive;
+import java.time.Duration;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -31,6 +32,7 @@ import org.apache.iceberg.FileFormat;
import org.apache.iceberg.Table;
import
org.apache.iceberg.relocated.com.google.common.util.concurrent.MoreExecutors;
import org.apache.iceberg.util.Tasks;
+import org.awaitility.Awaitility;
import org.junit.jupiter.api.Test;
import static org.apache.iceberg.TableProperties.COMMIT_MAX_RETRY_WAIT_MS;
@@ -55,19 +57,17 @@ public class TestHiveTableConcurrency extends
HiveTableBaseTest {
(ThreadPoolExecutor) Executors.newFixedThreadPool(2));
AtomicInteger barrier = new AtomicInteger(0);
- Tasks.range(2)
+ int threadsCount = 2;
+ Tasks.range(threadsCount)
.stopOnFailure().throwFailureWhenFinished()
.executeWith(executorService)
.run(index -> {
for (int numCommittedFiles = 0; numCommittedFiles < 10;
numCommittedFiles++) {
- while (barrier.get() < numCommittedFiles * 2) {
- try {
- Thread.sleep(10);
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
-
+ final int currentFilesCount = numCommittedFiles;
+ Awaitility.await()
+ .pollInterval(Duration.ofMillis(10))
+ .atMost(Duration.ofSeconds(10))
+ .until(() -> barrier.get() >= currentFilesCount *
threadsCount);
icebergTable.newFastAppend().appendFile(file).commit();
barrier.incrementAndGet();
}
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
index 84d9531b7ac..49c064db5d7 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
@@ -105,6 +105,7 @@ import org.apache.iceberg.expressions.Expression;
import org.apache.iceberg.expressions.Expressions;
import org.apache.iceberg.expressions.ResidualEvaluator;
import org.apache.iceberg.hive.CachedClientPool;
+import org.apache.iceberg.hive.HiveOperationsBase;
import org.apache.iceberg.hive.HiveSchemaUtil;
import org.apache.iceberg.hive.HiveTableOperations;
import org.apache.iceberg.hive.MetastoreLock;
@@ -1077,7 +1078,7 @@ public class HiveIcebergMetaHook implements HiveMetaHook {
hmsTable.getSd().getSerdeInfo().setSerializationLib(HiveIcebergSerDe.class.getName());
String storageHandler =
hmsTable.getParameters().get(hive_metastoreConstants.META_TABLE_STORAGE);
// Check if META_TABLE_STORAGE is not present or is not an instance of
ICEBERG_STORAGE_HANDLER
- if (storageHandler == null ||
!HiveTableOperations.isHiveIcebergStorageHandler(storageHandler)) {
+ if (storageHandler == null ||
!HiveOperationsBase.isHiveIcebergStorageHandler(storageHandler)) {
hmsTable.getParameters()
.put(hive_metastoreConstants.META_TABLE_STORAGE,
HiveTableOperations.HIVE_ICEBERG_STORAGE_HANDLER);
}
diff --git
a/iceberg/patched-iceberg-core/src/main/java/org/apache/iceberg/SerializableTable.java
b/iceberg/patched-iceberg-core/src/main/java/org/apache/iceberg/SerializableTable.java
index 14f5deadb91..63e5f1b52cd 100644
---
a/iceberg/patched-iceberg-core/src/main/java/org/apache/iceberg/SerializableTable.java
+++
b/iceberg/patched-iceberg-core/src/main/java/org/apache/iceberg/SerializableTable.java
@@ -22,6 +22,7 @@ package org.apache.iceberg;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
+import java.util.UUID;
import org.apache.hadoop.conf.Configuration;
import org.apache.iceberg.encryption.EncryptionManager;
import org.apache.iceberg.hadoop.HadoopConfigurable;
@@ -41,15 +42,15 @@ import org.apache.iceberg.util.SerializableSupplier;
* table metadata, it directly persists the current schema, spec, sort order,
table properties to
* avoid reading the metadata file from other nodes for frequently needed
metadata.
*
- * <p>The implementation assumes the passed instances of {@link FileIO},
{@link EncryptionManager},
- * {@link LocationProvider} are serializable. If you are serializing the table
using a custom
- * serialization framework like Kryo, those instances of {@link FileIO},
{@link EncryptionManager},
- * {@link LocationProvider} must be supported by that particular serialization
framework.
+ * <p>The implementation assumes the passed instances of {@link FileIO},
{@link EncryptionManager}
+ * are serializable. If you are serializing the table using a custom
serialization framework like
+ * Kryo, those instances of {@link FileIO}, {@link EncryptionManager} must be
supported by that
+ * particular serialization framework.
*
* <p><em>Note:</em> loading the complete metadata from a large number of
nodes can overwhelm the
* storage.
*/
-public class SerializableTable implements Table, Serializable {
+public class SerializableTable implements Table, HasTableOperations,
Serializable {
private final String name;
private final String location;
@@ -61,13 +62,14 @@ public class SerializableTable implements Table,
Serializable {
private final String sortOrderAsJson;
private final FileIO io;
private final EncryptionManager encryption;
- private final LocationProvider locationProvider;
private final Map<String, SnapshotRef> refs;
+ private transient volatile LocationProvider lazyLocationProvider = null;
private transient volatile Table lazyTable = null;
private transient volatile Schema lazySchema = null;
private transient volatile Map<Integer, PartitionSpec> lazySpecs = null;
private transient volatile SortOrder lazySortOrder = null;
+ private final UUID uuid;
protected SerializableTable(Table table) {
this.name = table.name();
@@ -82,8 +84,8 @@ public class SerializableTable implements Table, Serializable
{
this.sortOrderAsJson = SortOrderParser.toJson(table.sortOrder());
this.io = fileIO(table);
this.encryption = table.encryption();
- this.locationProvider = table.locationProvider();
this.refs = SerializableMap.copyOf(table.refs());
+ this.uuid = table.uuid();
}
/**
@@ -104,6 +106,8 @@ public class SerializableTable implements Table,
Serializable {
if (table instanceof HasTableOperations) {
TableOperations ops = ((HasTableOperations) table).operations();
return ops.current().metadataFileLocation();
+ } else if (table instanceof BaseMetadataTable) {
+ return ((BaseMetadataTable)
table).table().operations().current().metadataFileLocation();
} else {
return null;
}
@@ -127,7 +131,7 @@ public class SerializableTable implements Table,
Serializable {
}
TableOperations ops =
- new StaticTableOperations(metadataFileLocation, io,
locationProvider);
+ new StaticTableOperations(metadataFileLocation, io,
locationProvider());
this.lazyTable = newTable(ops, name);
}
}
@@ -235,7 +239,14 @@ public class SerializableTable implements Table,
Serializable {
@Override
public LocationProvider locationProvider() {
- return locationProvider;
+ if (lazyLocationProvider == null) {
+ synchronized (this) {
+ if (lazyLocationProvider == null) {
+ this.lazyLocationProvider = LocationProviders.locationsFor(location,
properties);
+ }
+ }
+ }
+ return lazyLocationProvider;
}
@Override
@@ -243,11 +254,21 @@ public class SerializableTable implements Table,
Serializable {
return lazyTable().statisticsFiles();
}
+ @Override
+ public List<PartitionStatisticsFile> partitionStatisticsFiles() {
+ return lazyTable().partitionStatisticsFiles();
+ }
+
@Override
public Map<String, SnapshotRef> refs() {
return refs;
}
+ @Override
+ public UUID uuid() {
+ return uuid;
+ }
+
@Override
public void refresh() {
throw new UnsupportedOperationException(errorMsg("refresh"));
@@ -352,6 +373,11 @@ public class SerializableTable implements Table,
Serializable {
throw new UnsupportedOperationException(errorMsg("updateStatistics"));
}
+ @Override
+ public UpdatePartitionStatistics updatePartitionStatistics() {
+ throw new
UnsupportedOperationException(errorMsg("updatePartitionStatistics"));
+ }
+
@Override
public ExpireSnapshots expireSnapshots() {
throw new UnsupportedOperationException(errorMsg("expireSnapshots"));
@@ -367,6 +393,11 @@ public class SerializableTable implements Table,
Serializable {
throw new UnsupportedOperationException(errorMsg("newTransaction"));
}
+ @Override
+ public StaticTableOperations operations() {
+ return (StaticTableOperations) ((BaseTable) lazyTable()).operations();
+ }
+
private String errorMsg(String operation) {
return String.format("Operation %s is not supported after the table is
serialized", operation);
}
diff --git a/iceberg/pom.xml b/iceberg/pom.xml
index dc9a842e543..eedbd7e5692 100644
--- a/iceberg/pom.xml
+++ b/iceberg/pom.xml
@@ -25,7 +25,7 @@
<properties>
<hive.path.to.root>..</hive.path.to.root>
<path.to.iceberg.root>.</path.to.iceberg.root>
- <iceberg.version>1.4.3</iceberg.version>
+ <iceberg.version>1.5.2</iceberg.version>
<kryo-shaded.version>4.0.3</kryo-shaded.version>
<iceberg.mockito-core.version>3.4.4</iceberg.mockito-core.version>
<iceberg.avro.version>1.11.3</iceberg.avro.version>
@@ -36,6 +36,7 @@
<google.errorprone.version>2.5.1</google.errorprone.version>
<assertj.version>3.24.2</assertj.version>
<junit.jupiter.version>5.10.0</junit.jupiter.version>
+ <awaitility.version>4.2.1</awaitility.version>
<immutables.value.version>2.10.0</immutables.value.version>
<validate.skip>false</validate.skip>
<caffeine.version>2.9.3</caffeine.version>
@@ -212,6 +213,11 @@
<artifactId>junit-jupiter-params</artifactId>
<version>${junit.jupiter.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.awaitility</groupId>
+ <artifactId>awaitility</artifactId>
+ <version>${awaitility.version}</version>
+ </dependency>
<dependency>
<groupId>com.esotericsoftware</groupId>
<artifactId>kryo-shaded</artifactId>