This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new a8e5073 HIVE-25710: Config used to enable non-blocking TRUNCATE is
not properly propagated (Denys Kuzmenko, reviewed by Karen Coppage and Peter
Vary)
a8e5073 is described below
commit a8e50734e0460e506f1762fbe0f628bcb444b8f5
Author: Denys Kuzmenko <dkuzme...@cloudera.com>
AuthorDate: Tue Nov 30 10:09:06 2021 +0200
HIVE-25710: Config used to enable non-blocking TRUNCATE is not properly
propagated (Denys Kuzmenko, reviewed by Karen Coppage and Peter Vary)
Closes #2796
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 11 +++++++--
.../table/misc/truncate/TruncateTableAnalyzer.java | 9 +++++---
.../org/apache/hadoop/hive/ql/metadata/Hive.java | 4 +++-
.../ql/metadata/SessionHiveMetaStoreClient.java | 12 ++++++++++
.../org/apache/hadoop/hive/ql/TestTxnCommands.java | 10 +++++++--
.../hadoop/hive/ql/TestTxnCommandsForMmTable.java | 14 ++++++++----
.../hadoop/hive/ql/lockmgr/TestDbTxnManager2.java | 2 +-
.../hadoop/hive/metastore/HiveMetaStoreClient.java | 26 +++++++++++++++-------
.../hadoop/hive/metastore/IMetaStoreClient.java | 2 ++
.../hadoop/hive/metastore/conf/MetastoreConf.java | 3 ---
.../apache/hadoop/hive/metastore/HMSHandler.java | 12 +++++-----
.../metastore/HiveMetaStoreClientPreCatalog.java | 7 ++++++
12 files changed, 81 insertions(+), 31 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index c0325a6..988cec8 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -3060,8 +3060,15 @@ public class HiveConf extends Configuration {
"Creates an _orc_acid_version file along with acid files, to store the
version data"),
HIVE_TXN_READONLY_ENABLED("hive.txn.readonly.enabled", false,
- "Enables read-only transaction classification and related
optimizations"),
-
+ "Enables read-only transaction classification and related
optimizations"),
+
+ HIVE_ACID_LOCKLESS_READS_ENABLED("hive.acid.lockless.reads.enabled", false,
+ "Enables lockless reads"),
+
+ HIVE_ACID_TRUNCATE_USE_BASE("hive.acid.truncate.usebase", false,
+ "If enabled, truncate for transactional tables will not delete the
data directories,\n" +
+ "rather create a new base directory with no datafiles."),
+
// Configs having to do with DeltaFilesMetricReporter, which collects
lists of most recently active tables
// with the most number of active/obsolete deltas.
HIVE_TXN_ACID_METRICS_MAX_CACHE_SIZE("hive.txn.acid.metrics.max.cache.size",
100,
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/truncate/TruncateTableAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/truncate/TruncateTableAnalyzer.java
index ab67994..47d39d4 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/truncate/TruncateTableAnalyzer.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/truncate/TruncateTableAnalyzer.java
@@ -116,10 +116,13 @@ public class TruncateTableAnalyzer extends
AbstractBaseAlterTableAnalyzer {
private void addTruncateTableOutputs(ASTNode root, Table table, Map<String,
String> partitionSpec)
throws SemanticException {
- boolean truncateKeepsDataFiles = AcidUtils.isTransactionalTable(table) &&
- MetastoreConf.getBoolVar(conf,
MetastoreConf.ConfVars.TRUNCATE_ACID_USE_BASE);
+ boolean truncateUseBase = (HiveConf.getBoolVar(conf,
HiveConf.ConfVars.HIVE_ACID_TRUNCATE_USE_BASE)
+ || HiveConf.getBoolVar(conf,
HiveConf.ConfVars.HIVE_ACID_LOCKLESS_READS_ENABLED))
+ && AcidUtils.isTransactionalTable(table);
+
WriteEntity.WriteType writeType =
- truncateKeepsDataFiles ? WriteEntity.WriteType.DDL_EXCL_WRITE :
WriteEntity.WriteType.DDL_EXCLUSIVE;
+ truncateUseBase ? WriteEntity.WriteType.DDL_EXCL_WRITE :
WriteEntity.WriteType.DDL_EXCLUSIVE;
+
if (partitionSpec == null) {
if (!table.isPartitioned()) {
outputs.add(new WriteEntity(table, writeType));
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 13c0514..b5a8d36 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -1433,8 +1433,10 @@ public class Hive {
if (snapshot == null) {
getMSC().truncateTable(table.getDbName(), table.getTableName(),
partNames);
} else {
+ boolean truncateUseBase = HiveConf.getBoolVar(conf,
HiveConf.ConfVars.HIVE_ACID_TRUNCATE_USE_BASE)
+ || HiveConf.getBoolVar(conf,
HiveConf.ConfVars.HIVE_ACID_LOCKLESS_READS_ENABLED);
getMSC().truncateTable(table.getDbName(), table.getTableName(),
partNames,
- snapshot.getValidWriteIdList(), snapshot.getWriteId());
+ snapshot.getValidWriteIdList(), snapshot.getWriteId(),
!truncateUseBase);
}
} catch (Exception e) {
throw new HiveException(e);
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
index 9bb648b..4744516 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
@@ -231,6 +231,18 @@ public class SessionHiveMetaStoreClient extends
HiveMetaStoreClientWithLocalCach
}
@Override
+ public void truncateTable(String dbName, String tableName,
+ List<String> partNames, String validWriteIds, long writeId, boolean
deleteData)
+ throws TException {
+ org.apache.hadoop.hive.metastore.api.Table table = getTempTable(dbName,
tableName);
+ if (table != null) {
+ truncateTempTable(table);
+ return;
+ }
+ super.truncateTable(dbName, tableName, partNames, validWriteIds, writeId,
deleteData);
+ }
+
+ @Override
public org.apache.hadoop.hive.metastore.api.Table getTable(String dbname,
String name) throws MetaException,
TException, NoSuchObjectException {
GetTableRequest getTableRequest = new GetTableRequest(dbname,name);
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
index b9fe07e..4c0b745 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
@@ -107,8 +107,8 @@ public class TestTxnCommands extends
TxnCommandsBaseForTests {
super.initHiveConf();
//TestTxnCommandsWithSplitUpdateAndVectorization has the vectorized version
//of these tests.
- hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, false);
- MetastoreConf.setBoolVar(hiveConf,
MetastoreConf.ConfVars.TRUNCATE_ACID_USE_BASE, true);
+ HiveConf.setBoolVar(hiveConf,
HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, false);
+ HiveConf.setBoolVar(hiveConf,
HiveConf.ConfVars.HIVE_ACID_TRUNCATE_USE_BASE, false);
}
@@ -1526,6 +1526,8 @@ public class TestTxnCommands extends
TxnCommandsBaseForTests {
@Test
public void testTruncateWithBase() throws Exception{
+ HiveConf.setBoolVar(hiveConf,
HiveConf.ConfVars.HIVE_ACID_TRUNCATE_USE_BASE, true);
+
runStatementOnDriver("insert into " + Table.ACIDTBL + "
values(1,2),(3,4)");
runStatementOnDriver("truncate table " + Table.ACIDTBL);
@@ -1544,6 +1546,8 @@ public class TestTxnCommands extends
TxnCommandsBaseForTests {
@Test
public void testTruncateWithBaseAllPartition() throws Exception{
+ HiveConf.setBoolVar(hiveConf,
HiveConf.ConfVars.HIVE_ACID_TRUNCATE_USE_BASE, true);
+
runStatementOnDriver("insert into " + Table.ACIDTBLPART + "
partition(p='a') values(1,2),(3,4)");
runStatementOnDriver("insert into " + Table.ACIDTBLPART + "
partition(p='b') values(1,2),(3,4)");
runStatementOnDriver("truncate table " + Table.ACIDTBLPART);
@@ -1563,6 +1567,8 @@ public class TestTxnCommands extends
TxnCommandsBaseForTests {
@Test
public void testTruncateWithBaseOnePartition() throws Exception{
+ HiveConf.setBoolVar(hiveConf,
HiveConf.ConfVars.HIVE_ACID_TRUNCATE_USE_BASE, true);
+
runStatementOnDriver("insert into " + Table.ACIDTBLPART + "
partition(p='a') values(1,2),(3,4)");
runStatementOnDriver("insert into " + Table.ACIDTBLPART + "
partition(p='b') values(5,5),(4,4)");
runStatementOnDriver("truncate table " + Table.ACIDTBLPART + "
partition(p='b')");
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForMmTable.java
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForMmTable.java
index 2476f00..cba9ad2 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForMmTable.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForMmTable.java
@@ -80,7 +80,7 @@ public class TestTxnCommandsForMmTable extends
TxnCommandsBaseForTests {
@Override
void initHiveConf() {
super.initHiveConf();
- MetastoreConf.setBoolVar(hiveConf,
MetastoreConf.ConfVars.TRUNCATE_ACID_USE_BASE, true);
+ HiveConf.setBoolVar(hiveConf,
HiveConf.ConfVars.HIVE_ACID_TRUNCATE_USE_BASE, false);
}
@Override
@@ -596,7 +596,9 @@ public class TestTxnCommandsForMmTable extends
TxnCommandsBaseForTests {
}
@Test
- public void testTruncateWithBase() throws Exception{
+ public void testTruncateWithBase() throws Exception {
+ HiveConf.setBoolVar(hiveConf,
HiveConf.ConfVars.HIVE_ACID_TRUNCATE_USE_BASE, true);
+
runStatementOnDriver("insert into " + TableExtended.MMTBL + "
values(1,2),(3,4)");
runStatementOnDriver("truncate table " + TableExtended.MMTBL);
@@ -614,7 +616,9 @@ public class TestTxnCommandsForMmTable extends
TxnCommandsBaseForTests {
}
@Test
- public void testTruncateWithBaseAllPartition() throws Exception{
+ public void testTruncateWithBaseAllPartition() throws Exception {
+ HiveConf.setBoolVar(hiveConf,
HiveConf.ConfVars.HIVE_ACID_TRUNCATE_USE_BASE, true);
+
runStatementOnDriver("insert into " + TableExtended.MMTBLPART + "
partition(p='a') values(1,2),(3,4)");
runStatementOnDriver("insert into " + TableExtended.MMTBLPART + "
partition(p='b') values(1,2),(3,4)");
runStatementOnDriver("truncate table " + TableExtended.MMTBLPART);
@@ -633,7 +637,9 @@ public class TestTxnCommandsForMmTable extends
TxnCommandsBaseForTests {
}
@Test
- public void testTruncateWithBaseOnePartition() throws Exception{
+ public void testTruncateWithBaseOnePartition() throws Exception {
+ HiveConf.setBoolVar(hiveConf,
HiveConf.ConfVars.HIVE_ACID_TRUNCATE_USE_BASE, true);
+
runStatementOnDriver("insert into " + TableExtended.MMTBLPART + "
partition(p='a') values(1,2),(3,4)");
runStatementOnDriver("insert into " + TableExtended.MMTBLPART+ "
partition(p='b') values(5,5),(4,4)");
runStatementOnDriver("truncate table " + TableExtended.MMTBLPART + "
partition(p='b')");
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
index 43a78e0..e17150f 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
@@ -3178,7 +3178,7 @@ public class TestDbTxnManager2 extends
DbTxnManagerEndToEndTestBase{
}
private void testTruncate(boolean useBaseDir) throws Exception {
- MetastoreConf.setBoolVar(conf,
MetastoreConf.ConfVars.TRUNCATE_ACID_USE_BASE, useBaseDir);
+ HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_ACID_TRUNCATE_USE_BASE,
useBaseDir);
dropTable(new String[] {"T"});
driver.run("create table T (a int, b int) stored as orc
tblproperties('transactional'='true')");
driver.run("insert into T values(0,2),(1,4)");
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index d9aa2f5..58c37b2 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -104,7 +104,9 @@ import com.google.common.collect.Lists;
public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
private final String CLASS_NAME = HiveMetaStoreClient.class.getName();
+
public static final String MANUALLY_INITIATED_COMPACTION = "manual";
+ public static final String TRUNCATE_SKIP_DATA_DELETION =
"truncateSkipDataDeletion";
/**
* Capabilities of the current client. If this client talks to a MetaStore
server in a manner
@@ -1843,37 +1845,45 @@ public class HiveMetaStoreClient implements
IMetaStoreClient, AutoCloseable {
@Override
public void truncateTable(String dbName, String tableName, List<String>
partNames,
+ String validWriteIds, long writeId, boolean deleteData) throws
TException {
+ truncateTableInternal(getDefaultCatalog(conf),
+ dbName, tableName, partNames, validWriteIds, writeId, deleteData);
+ }
+
+ @Override
+ public void truncateTable(String dbName, String tableName, List<String>
partNames,
String validWriteIds, long writeId) throws TException {
truncateTableInternal(getDefaultCatalog(conf),
- dbName, tableName, partNames, validWriteIds, writeId);
+ dbName, tableName, partNames, validWriteIds, writeId, true);
}
@Override
public void truncateTable(String dbName, String tableName, List<String>
partNames) throws TException {
- truncateTableInternal(getDefaultCatalog(conf), dbName, tableName,
partNames, null, -1);
+ truncateTableInternal(getDefaultCatalog(conf), dbName, tableName,
partNames, null, -1, true);
}
@Override
public void truncateTable(String catName, String dbName, String tableName,
List<String> partNames)
throws TException {
- truncateTableInternal(catName, dbName, tableName, partNames, null, -1);
+ truncateTableInternal(catName, dbName, tableName, partNames, null, -1,
true);
}
private void truncateTableInternal(String catName, String dbName, String
tableName,
- List<String> partNames, String validWriteIds, long writeId)
- throws MetaException, TException {
+ List<String> partNames, String validWriteIds, long writeId, boolean
deleteData)
+ throws TException {
Table table = getTable(catName, dbName, tableName);
HiveMetaHook hook = getHook(table);
- EnvironmentContext envContext = new EnvironmentContext();
+ EnvironmentContext context = new EnvironmentContext();
+ context.putToProperties(TRUNCATE_SKIP_DATA_DELETION,
Boolean.toString(!deleteData));
if (hook != null) {
- hook.preTruncateTable(table, envContext);
+ hook.preTruncateTable(table, context);
}
TruncateTableRequest req = new TruncateTableRequest(
prependCatalogToDbName(catName, dbName, conf), tableName);
req.setPartNames(partNames);
req.setValidWriteIdList(validWriteIds);
req.setWriteId(writeId);
- req.setEnvironmentContext(envContext);
+ req.setEnvironmentContext(context);
client.truncate_table_req(req);
}
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
index 294aac9..dc1eb2e 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
@@ -560,6 +560,8 @@ public interface IMetaStoreClient {
void truncateTable(String dbName, String tableName, List<String> partNames,
String validWriteIds, long writeId) throws TException;
+ void truncateTable(String dbName, String tableName, List<String> partNames,
+ String validWriteIds, long writeId, boolean deleteData) throws
TException;
/**
* Truncate the table/partitions in the DEFAULT database.
* @param catName catalog name
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 1738ab4..448ea6a 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -1400,9 +1400,6 @@ public class MetastoreConf {
" If org.apache.hive.hcatalog.listener.DbNotificationListener is
configured along with other transactional event" +
" listener implementation classes, make sure
org.apache.hive.hcatalog.listener.DbNotificationListener is placed at" +
" the end of the list."),
- TRUNCATE_ACID_USE_BASE("metastore.acid.truncate.usebase",
"hive.metastore.acid.truncate.usebase", false,
- "If enabled, truncate for transactional tables will not delete the
data directories,\n" +
- "rather create a new base directory with no datafiles."),
TRY_DIRECT_SQL("metastore.try.direct.sql",
"hive.metastore.try.direct.sql", true,
"Whether the metastore should try to use direct SQL queries instead of
the\n" +
"DataNucleus for certain read paths. This can improve metastore
performance when\n" +
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
index deae25d..70ca184 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
@@ -104,6 +104,7 @@ import java.util.regex.Pattern;
import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.apache.commons.lang3.StringUtils.join;
+import static
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.TRUNCATE_SKIP_DATA_DELETION;
import static
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.TABLE_IS_CTAS;
import static
org.apache.hadoop.hive.metastore.ExceptionHandler.handleException;
import static
org.apache.hadoop.hive.metastore.ExceptionHandler.newMetaException;
@@ -153,7 +154,6 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
@VisibleForTesting
static long testTimeoutValue = -1;
- public static final String TRUNCATE_SKIP_DATA_DELETION =
"truncateSkipDataDeletion";
public static final String ADMIN = "admin";
public static final String PUBLIC = "public";
@@ -3410,12 +3410,10 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
.map(Boolean::parseBoolean)
.orElse(false);
- if (!skipDataDeletion) {
- boolean truncateFiles = !TxnUtils.isTransactionalTable(tbl)
- || !MetastoreConf.getBoolVar(getConf(),
MetastoreConf.ConfVars.TRUNCATE_ACID_USE_BASE);
-
- if (truncateFiles) {
+ if (TxnUtils.isTransactionalTable(tbl) || !skipDataDeletion) {
+ if (!skipDataDeletion) {
isSkipTrash = MetaStoreUtils.isSkipTrash(tbl.getParameters());
+
Database db = get_database_core(parsedDbName[CAT_NAME],
parsedDbName[DB_NAME]);
needCmRecycle = ReplChangeManager.shouldEnableCm(db, tbl);
}
@@ -3423,7 +3421,7 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
for (Path location : getLocationsForTruncate(getMS(),
parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName,
tbl, partNames)) {
FileSystem fs = location.getFileSystem(getConf());
- if (truncateFiles) {
+ if (!skipDataDeletion) {
truncateDataFiles(location, fs, isSkipTrash, needCmRecycle);
} else {
// For Acid tables we don't need to delete the old files, only
write an empty baseDir.
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
index a12662e..181b415 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
@@ -3878,6 +3878,13 @@ public class HiveMetaStoreClientPreCatalog implements
IMetaStoreClient, AutoClos
}
@Override
+ public void truncateTable(String dbName, String tableName,
+ List<String> partNames, String validWriteIds, long writeId, boolean
deleteData)
+ throws TException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public GetPartitionsResponse getPartitionsWithSpecs(GetPartitionsRequest
request)
throws TException {
throw new UnsupportedOperationException();
