This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 4d9fdf211a7 HIVE-27266: Retrieve only partNames if not need drop data
in HMSHandler.dropPartitionsAndGetLocations (Wechar Yu, reviewed by Attila
Turoczy, Denys Kuzmenko)
4d9fdf211a7 is described below
commit 4d9fdf211a71f67457302446d78f1183a44f074d
Author: Wechar Yu <yuwq1...@gmail.com>
AuthorDate: Thu Apr 27 23:05:25 2023 +0800
HIVE-27266: Retrieve only partNames if not need drop data in
HMSHandler.dropPartitionsAndGetLocations (Wechar Yu, reviewed by Attila
Turoczy, Denys Kuzmenko)
Closes #4238
---
.../apache/hadoop/hive/metastore/HMSHandler.java | 24 ++++++++++++---------
.../hadoop/hive/metastore/tools/BenchmarkTool.java | 5 +++++
.../hadoop/hive/metastore/tools/HMSBenchmarks.java | 25 ++++++++++++++++++++++
.../hadoop/hive/metastore/tools/HMSClient.java | 7 +++++-
4 files changed, 50 insertions(+), 11 deletions(-)
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
index 15bb8d82245..308396ef3fe 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
@@ -3132,15 +3132,12 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
List<Path> partPaths = new ArrayList<>();
while (true) {
- Map<String, String> partitionLocations =
ms.getPartitionLocations(catName, dbName, tableName,
- tableDnsPath, batchSize);
- if (partitionLocations == null || partitionLocations.isEmpty()) {
- // No more partitions left to drop. Return with the collected path
list to delete.
- return partPaths;
- }
-
+ List<String> partNames;
if (checkLocation) {
- for (String partName : partitionLocations.keySet()) {
+ Map<String, String> partitionLocations =
ms.getPartitionLocations(catName, dbName, tableName,
+ tableDnsPath, batchSize);
+ partNames = new ArrayList<>(partitionLocations.keySet());
+ for (String partName : partNames) {
String pathString = partitionLocations.get(partName);
if (pathString != null) {
Path partPath = wh.getDnsPath(new Path(pathString));
@@ -3157,19 +3154,26 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
}
}
}
+ } else {
+ partNames = ms.listPartitionNames(catName, dbName, tableName, (short)
batchSize);
+ }
+
+ if (partNames == null || partNames.isEmpty()) {
+ // No more partitions left to drop. Return with the collected path
list to delete.
+ return partPaths;
}
for (MetaStoreEventListener listener : listeners) {
//No drop part listener events fired for public listeners
historically, for drop table case.
//Limiting to internal listeners for now, to avoid unexpected calls
for public listeners.
if (listener instanceof HMSMetricsListener) {
- for (@SuppressWarnings("unused") String partName :
partitionLocations.keySet()) {
+ for (@SuppressWarnings("unused") String partName : partNames) {
listener.onDropPartition(null);
}
}
}
- ms.dropPartitions(catName, dbName, tableName, new
ArrayList<>(partitionLocations.keySet()));
+ ms.dropPartitions(catName, dbName, tableName, partNames);
}
}
diff --git
a/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkTool.java
b/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkTool.java
index 943b87e9c97..025b39339bc 100644
---
a/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkTool.java
+++
b/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkTool.java
@@ -48,6 +48,7 @@ import static
org.apache.hadoop.hive.metastore.tools.Constants.HMS_DEFAULT_PORT;
import static
org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkCreatePartition;
import static
org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkCreatePartitions;
import static
org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkDeleteCreate;
+import static
org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkDeleteMetaOnlyWithPartitions;
import static
org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkDeleteWithPartitions;
import static
org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkDropDatabase;
import static
org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkDropPartition;
@@ -271,6 +272,8 @@ public class BenchmarkTool implements Runnable {
.add("dropTable", () -> benchmarkDeleteCreate(bench, bData))
.add("dropTableWithPartitions",
() -> benchmarkDeleteWithPartitions(bench, bData, 1,
nParameters[0]))
+ .add("dropTableMetadataWithPartitions",
+ () -> benchmarkDeleteMetaOnlyWithPartitions(bench, bData, 1,
nParameters[0]))
.add("addPartition", () -> benchmarkCreatePartition(bench, bData))
.add("dropPartition", () -> benchmarkDropPartition(bench, bData, 1))
.add("listPartition", () -> benchmarkListPartition(bench, bData))
@@ -294,6 +297,8 @@ public class BenchmarkTool implements Runnable {
() -> benchmarkListTables(bench, bData, howMany))
.add("dropTableWithPartitions" + '.' + howMany,
() -> benchmarkDeleteWithPartitions(bench, bData, howMany,
nParameters[0]))
+ .add("dropTableMetaOnlyWithPartitions" + '.' + howMany,
+ () -> benchmarkDeleteMetaOnlyWithPartitions(bench, bData,
howMany, nParameters[0]))
.add("listPartitions" + '.' + howMany,
() -> benchmarkListManyPartitions(bench, bData, howMany))
.add("getPartitions" + '.' + howMany,
diff --git
a/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSBenchmarks.java
b/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSBenchmarks.java
index 3363b9d7ff7..c48ab7d2b17 100644
---
a/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSBenchmarks.java
+++
b/standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSBenchmarks.java
@@ -123,6 +123,31 @@ final class HMSBenchmarks {
null);
}
+ static DescriptiveStatistics benchmarkDeleteMetaOnlyWithPartitions(@NotNull
MicroBenchmark bench,
+
@NotNull BenchData data,
+ int
howMany,
+ int
nparams) {
+ final HMSClient client = data.getClient();
+ String dbName = data.dbName;
+ String tableName = data.tableName;
+
+ // Create many parameters
+ Map<String, String> parameters = new HashMap<>(nparams);
+ for (int i = 0; i < nparams; i++) {
+ parameters.put(PARAM_KEY + i, PARAM_VALUE + i);
+ }
+
+ return bench.measure(
+ () -> throwingSupplierWrapper(() -> {
+ BenchmarkUtils.createPartitionedTable(client, dbName, tableName);
+ addManyPartitions(client, dbName, tableName, parameters,
+ Collections.singletonList("d"), howMany);
+ return true;
+ }),
+ () -> throwingSupplierWrapper(() -> client.dropTable(dbName,
tableName, false)),
+ null);
+ }
+
static DescriptiveStatistics benchmarkGetTable(@NotNull MicroBenchmark bench,
@NotNull BenchData data) {
final HMSClient client = data.getClient();
diff --git
a/standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSClient.java
b/standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSClient.java
index afe847281e4..c7a0093628f 100644
---
a/standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSClient.java
+++
b/standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSClient.java
@@ -251,7 +251,12 @@ final class HMSClient implements AutoCloseable {
}
boolean dropTable(@NotNull String dbName, @NotNull String tableName) throws
TException {
- client.drop_table(dbName, tableName, true);
+ return dropTable(dbName, tableName, true);
+ }
+
+ boolean dropTable(@NotNull String dbName, @NotNull String tableName, boolean
deleteData)
+ throws TException {
+ client.drop_table(dbName, tableName, deleteData);
return true;
}
