This is an automated email from the ASF dual-hosted git repository.
deniskuzZ pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 02004f8dab1 HIVE-30231: Revert the Iceberg part of HIVE-29244 (#6932)
02004f8dab1 is described below
commit 02004f8dab1039ef1b5258b7c52c9e8909377a28
Author: Neeraj Khatri <[email protected]>
AuthorDate: Sat Oct 31 22:03:20 2026 +0530
HIVE-30231: Revert the Iceberg part of HIVE-29244 (#6932)
---
.../org/apache/iceberg/hive/MetastoreLock.java | 53 ++++++++--------------
.../apache/iceberg/hive/TestHiveCommitLocks.java | 41 +++++++++++++----
2 files changed, 53 insertions(+), 41 deletions(-)
diff --git
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreLock.java
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreLock.java
index 465ecaa56e7..c0d9d88ee9f 100644
---
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreLock.java
+++
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreLock.java
@@ -78,7 +78,6 @@ public class MetastoreLock implements HiveLock {
private final ClientPool<IMetaStoreClient, TException> metaClients;
- private final String catalogName;
private final String databaseName;
private final String tableName;
private final String fullName;
@@ -101,7 +100,6 @@ public MetastoreLock(Configuration conf,
ClientPool<IMetaStoreClient, TException
String catalogName, String databaseName, String
tableName) {
this.metaClients = metaClients;
this.fullName = catalogName + "." + databaseName + "." + tableName;
- this.catalogName = catalogName;
this.databaseName = databaseName;
this.tableName = tableName;
@@ -181,8 +179,8 @@ public void unlock() {
@SuppressWarnings("checkstyle:CyclomaticComplexity")
private long acquireLock() throws LockException {
if (hmsLockId.isPresent()) {
- throw new IllegalArgumentException(String.format("HMS lock ID=%s already
acquired for table %s.%s.%s",
- hmsLockId.get(), catalogName, databaseName, tableName));
+ throw new IllegalArgumentException(String.format("HMS lock ID=%s already
acquired for table %s.%s",
+ hmsLockId.get(), databaseName, tableName));
}
LockInfo lockInfo = createLock();
@@ -214,13 +212,12 @@ private long acquireLock() throws LockException {
lockInfo.lockState = newState;
if (newState.equals(LockState.WAITING)) {
throw new WaitingForLockException(String.format(
- "Waiting for lock on table %s.%s.%s", catalogName,
databaseName, tableName));
+ "Waiting for lock on table %s.%s", databaseName,
tableName));
}
} catch (InterruptedException e) {
Thread.interrupted(); // Clear the interrupt status flag
LOG.warn(
- "Interrupted while waiting for lock on table {}.{}.{}",
- catalogName,
+ "Interrupted while waiting for lock on table {}.{}",
databaseName,
tableName,
e);
@@ -242,19 +239,19 @@ private long acquireLock() throws LockException {
if (!lockInfo.lockState.equals(LockState.ACQUIRED)) {
// timeout and do not have lock acquired
if (timeout) {
- throw new LockException("Timed out after %s ms waiting for lock on
%s.%s.%s",
- duration, catalogName, databaseName, tableName);
+ throw new LockException("Timed out after %s ms waiting for lock on
%s.%s",
+ duration, databaseName, tableName);
}
if (thriftError != null) {
throw new LockException(
- thriftError, "Metastore operation failed for %s.%s.%s",
catalogName, databaseName, tableName);
+ thriftError, "Metastore operation failed for %s.%s",
databaseName, tableName);
}
// Just for safety. We should not get here.
throw new LockException(
- "Could not acquire the lock on %s.%s.%s, lock request ended in
state %s",
- catalogName, databaseName, tableName, lockInfo.lockState);
+ "Could not acquire the lock on %s.%s, lock request ended in
state %s",
+ databaseName, tableName, lockInfo.lockState);
} else {
return lockInfo.lockId;
}
@@ -280,7 +277,6 @@ private LockInfo createLock() throws LockException {
LockComponent lockComponent =
new LockComponent(LockType.EXCL_WRITE, LockLevel.TABLE,
databaseName);
- lockComponent.setCatName(catalogName);
lockComponent.setTablename(tableName);
LockRequest lockRequest =
new LockRequest(
@@ -322,32 +318,27 @@ private LockInfo createLock() throws LockException {
}
}
- throw new LockException("Failed to find lock for table
%s.%s.%s", catalogName, databaseName,
- tableName);
+ throw new LockException("Failed to find lock for table
%s.%s", databaseName, tableName);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
interrupted.set(true);
LOG.warn(
- "Interrupted while trying to find lock for table
{}.{}.{}", catalogName, databaseName,
- tableName, e);
+ "Interrupted while trying to find lock for table
{}.{}", databaseName, tableName, e);
throw new LockException(
- e, "Interrupted while trying to find lock for
table %s.%s.%s", catalogName, databaseName,
- tableName);
+ e, "Interrupted while trying to find lock for
table %s.%s", databaseName, tableName);
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
interrupted.set(true);
- LOG.warn("Interrupted while creating lock on table
{}.{}.{}", catalogName, databaseName,
- tableName, e);
+ LOG.warn("Interrupted while creating lock on table {}.{}",
databaseName, tableName, e);
throw new LockException(
- e, "Interrupted while creating lock on table
%s.%s.%s", catalogName, databaseName,
- tableName);
+ e, "Interrupted while creating lock on table
%s.%s", databaseName, tableName);
}
},
LockException.class);
// This should be initialized always, or exception should be thrown.
- LOG.debug("Lock {} created for table {}.{}.{}", lockInfo, catalogName,
databaseName, tableName);
+ LOG.debug("Lock {} created for table {}.{}", lockInfo, databaseName,
tableName);
return lockInfo;
}
@@ -363,14 +354,13 @@ private LockInfo findLock() throws LockException,
InterruptedException {
HiveVersion.min(HiveVersion.HIVE_2),
"Minimally Hive 2 HMS client is needed to find the Lock using the
showLocks API call");
ShowLocksRequest showLocksRequest = new ShowLocksRequest();
- showLocksRequest.setCatname(catalogName);
showLocksRequest.setDbname(databaseName);
showLocksRequest.setTablename(tableName);
ShowLocksResponse response;
try {
response = metaClients.run(client -> client.showLocks(showLocksRequest));
} catch (TException e) {
- throw new LockException(e, "Failed to find lock for table %s.%s.%s",
catalogName, databaseName, tableName);
+ throw new LockException(e, "Failed to find lock for table %s.%s",
databaseName, tableName);
}
for (ShowLocksResponseElement lock : response.getLocks()) {
if (lock.getAgentInfo().equals(agentInfo)) {
@@ -413,22 +403,19 @@ private void unlock(Optional<Long> lockId) {
// Interrupted unlock. We try to unlock one more time if we have a
lockId
try {
Thread.interrupted(); // Clear the interrupt status flag for now, so
we can retry unlock
- LOG.warn("Interrupted unlock we try one more time {}.{}.{}",
catalogName, databaseName,
- tableName, ie);
+ LOG.warn("Interrupted unlock we try one more time {}.{}",
databaseName, tableName, ie);
doUnlock(id);
} catch (Exception e) {
- LOG.warn("Failed to unlock even on 2nd attempt {}.{}.{}",
catalogName, databaseName,
- tableName, e);
+ LOG.warn("Failed to unlock even on 2nd attempt {}.{}", databaseName,
tableName, e);
} finally {
Thread.currentThread().interrupt(); // Set back the interrupt status
}
} else {
Thread.currentThread().interrupt(); // Set back the interrupt status
- LOG.warn("Interrupted finding locks to unlock {}.{}.{}", catalogName,
databaseName,
- tableName, ie);
+ LOG.warn("Interrupted finding locks to unlock {}.{}", databaseName,
tableName, ie);
}
} catch (Exception e) {
- LOG.warn("Failed to unlock {}.{}.{}", catalogName, databaseName,
tableName, e);
+ LOG.warn("Failed to unlock {}.{}", databaseName, tableName, e);
}
}
diff --git
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveCommitLocks.java
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveCommitLocks.java
index fb08127683b..c7cbbda5f75 100644
---
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveCommitLocks.java
+++
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveCommitLocks.java
@@ -32,9 +32,12 @@
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
+import org.apache.hadoop.hive.metastore.api.LockComponent;
+import org.apache.hadoop.hive.metastore.api.LockLevel;
import org.apache.hadoop.hive.metastore.api.LockRequest;
import org.apache.hadoop.hive.metastore.api.LockResponse;
import org.apache.hadoop.hive.metastore.api.LockState;
+import org.apache.hadoop.hive.metastore.api.LockType;
import org.apache.hadoop.hive.metastore.api.ShowLocksResponse;
import org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement;
import org.apache.iceberg.CatalogProperties;
@@ -371,7 +374,7 @@ public void testUnLockAfterInterruptedLock() throws
TException {
.isInstanceOf(RuntimeException.class)
.hasMessage(
"org.apache.iceberg.hive.LockException: " +
- "Interrupted while creating lock on table hive.hivedb.tbl");
+ "Interrupted while creating lock on table hivedb.tbl");
verify(spyClient, times(1)).unlock(eq(dummyLockId));
// Make sure that we exit the lock loop on InterruptedException
@@ -395,7 +398,7 @@ public void testUnLockAfterInterruptedLockCheck() throws
TException {
.isInstanceOf(RuntimeException.class)
.hasMessage(
"org.apache.iceberg.hive.LockException: " +
- "Could not acquire the lock on hive.hivedb.tbl, lock request
ended in state WAITING");
+ "Could not acquire the lock on hivedb.tbl, lock request ended
in state WAITING");
verify(spyClient, times(1)).unlock(eq(dummyLockId));
// Make sure that we exit the checkLock loop on InterruptedException
@@ -454,7 +457,7 @@ public void testLockFailureAtFirstTime() throws TException {
.isInstanceOf(CommitFailedException.class)
.hasMessage(
"org.apache.iceberg.hive.LockException: " +
- "Could not acquire the lock on hive.hivedb.tbl, lock request
ended in state NOT_ACQUIRED");
+ "Could not acquire the lock on hivedb.tbl, lock request ended
in state NOT_ACQUIRED");
}
@Test
@@ -472,7 +475,7 @@ public void testLockFailureAfterRetries() throws TException
{
.isInstanceOf(CommitFailedException.class)
.hasMessage(
"org.apache.iceberg.hive.LockException: " +
- "Could not acquire the lock on hive.hivedb.tbl, lock request
ended in state NOT_ACQUIRED");
+ "Could not acquire the lock on hivedb.tbl, lock request ended
in state NOT_ACQUIRED");
}
@Test
@@ -484,7 +487,7 @@ public void testLockTimeoutAfterRetries() throws TException
{
.isInstanceOf(CommitFailedException.class)
.hasMessageStartingWith("org.apache.iceberg.hive.LockException")
.hasMessageContaining("Timed out after")
- .hasMessageEndingWith("waiting for lock on hive.hivedb.tbl");
+ .hasMessageEndingWith("waiting for lock on hivedb.tbl");
}
@Test
@@ -509,7 +512,7 @@ public void
testPassThroughThriftExceptionsForHiveVersion_1()
assertThatThrownBy(() -> spyOps.doCommit(metadataV2, metadataV1))
.isInstanceOf(CommitFailedException.class)
.hasMessage(
- "org.apache.iceberg.hive.LockException: Failed to find lock for
table hive.hivedb.tbl");
+ "org.apache.iceberg.hive.LockException: Failed to find lock for
table hivedb.tbl");
}
}
@@ -522,7 +525,7 @@ public void testPassThroughThriftExceptions() throws
TException {
assertThatThrownBy(() -> spyOps.doCommit(metadataV2, metadataV1))
.isInstanceOf(RuntimeException.class)
.hasMessage(
- "org.apache.iceberg.hive.LockException: Metastore operation failed
for hive.hivedb.tbl");
+ "org.apache.iceberg.hive.LockException: Metastore operation failed
for hivedb.tbl");
}
@Test
@@ -538,7 +541,7 @@ public void testPassThroughInterruptions() throws
TException {
.isInstanceOf(CommitFailedException.class)
.hasMessage(
"org.apache.iceberg.hive.LockException: " +
- "Could not acquire the lock on hive.hivedb.tbl, lock request
ended in state WAITING");
+ "Could not acquire the lock on hivedb.tbl, lock request ended
in state WAITING");
}
@Test
@@ -635,4 +638,26 @@ public void testNoLockCallsWithNoLock() throws TException {
assertThat(METADATA_LOCATION_PROP).isEqualTo(context.get("expected_parameter_key"));
assertThat(metadataV2.metadataFileLocation()).isEqualTo(context.get("expected_parameter_value"));
}
+
+ @Test
+ public void testLockConflictsWithOtherEnginesWhateverTheCatalogName() throws
Exception {
+ // The Iceberg catalog name is not an HMS catalog; other engines lock
without a catalog
+ MetastoreLock icebergLock = new MetastoreLock(overriddenHiveConf,
spyCachedClientPool,
+ IcebergCatalogProperties.ICEBERG_DEFAULT_CATALOG_NAME, DB_NAME,
TABLE_NAME);
+ icebergLock.lock();
+ try {
+ LockComponent otherEngineComponent = new
LockComponent(LockType.EXCL_WRITE, LockLevel.TABLE, DB_NAME);
+ otherEngineComponent.setTablename(TABLE_NAME);
+ LockRequest otherEngineRequest = new LockRequest(
+ Lists.newArrayList(otherEngineComponent), "otherEngineUser",
"localhost");
+ LockResponse otherEngineResponse = spyClient.lock(otherEngineRequest);
+ try {
+
assertThat(otherEngineResponse.getState()).isEqualTo(LockState.WAITING);
+ } finally {
+ spyClient.unlock(otherEngineResponse.getLockid());
+ }
+ } finally {
+ icebergLock.unlock();
+ }
+ }
}