This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 3d2ac60baec [bugfix](hive)Delete the temporarily created folder
(#40424) (#40722)
3d2ac60baec is described below
commit 3d2ac60baec3f4c6509d8f66c83e5afcda344095
Author: wuwenchi <[email protected]>
AuthorDate: Thu Sep 12 19:18:35 2024 +0800
[bugfix](hive)Delete the temporarily created folder (#40424) (#40722)
## Proposed changes
bp: #40424
---
.../doris/datasource/hive/HMSTransaction.java | 2 ++
.../doris/datasource/hive/HmsCommitTest.java | 22 ++++++++++++++++++++--
2 files changed, 22 insertions(+), 2 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java
index 085657c11ac..290fd397113 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java
@@ -1443,6 +1443,8 @@ public class HMSTransaction implements Transaction {
runS3cleanWhenSuccess();
doAddPartitionsTask();
doUpdateStatisticsTasks();
+ //delete write path
+ pruneAndDeleteStagingDirectories();
doNothing();
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HmsCommitTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HmsCommitTest.java
index 7e99667b731..395a063fbc8 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HmsCommitTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HmsCommitTest.java
@@ -17,6 +17,7 @@
package org.apache.doris.datasource.hive;
+import org.apache.doris.backup.Status;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.info.SimpleTableInfo;
@@ -169,15 +170,21 @@ public class HmsCommitTest {
@Test
public void testAppendPartitionForUnPartitionedTable() throws IOException {
genQueryID();
- System.out.println(DebugUtil.printId(connectContext.queryId()));
List<THivePartitionUpdate> pus = new ArrayList<>();
pus.add(createRandomAppend(null));
pus.add(createRandomAppend(null));
pus.add(createRandomAppend(null));
+ new
MockUp<HMSTransaction.HmsCommitter>(HMSTransaction.HmsCommitter.class) {
+ @Mock
+ private void doNothing() {
+ Assert.assertEquals(Status.ErrCode.NOT_FOUND,
fs.exists(getWritePath()).getErrCode());
+ }
+ };
commit(dbName, tbWithoutPartition, pus);
Table table = hmsClient.getTable(dbName, tbWithoutPartition);
assertNumRows(3, table);
+
genQueryID();
List<THivePartitionUpdate> pus2 = new ArrayList<>();
pus2.add(createRandomAppend(null));
@@ -204,6 +211,12 @@ public class HmsCommitTest {
@Test
public void testNewPartitionForPartitionedTable() throws IOException {
+ new
MockUp<HMSTransaction.HmsCommitter>(HMSTransaction.HmsCommitter.class) {
+ @Mock
+ private void doNothing() {
+ Assert.assertEquals(Status.ErrCode.NOT_FOUND,
fs.exists(getWritePath()).getErrCode());
+ }
+ };
genQueryID();
List<THivePartitionUpdate> pus = new ArrayList<>();
pus.add(createRandomNew("a"));
@@ -377,6 +390,11 @@ public class HmsCommitTest {
genOnePartitionUpdate("c3=" + partition,
TUpdateMode.OVERWRITE);
}
+ private String getWritePath() {
+ String queryId = DebugUtil.printId(ConnectContext.get().queryId());
+ return writeLocation + queryId + "/";
+ }
+
public void commit(String dbName,
String tableName,
List<THivePartitionUpdate> hivePUs) {
@@ -385,7 +403,7 @@ public class HmsCommitTest {
HiveInsertCommandContext ctx = new HiveInsertCommandContext();
String queryId = DebugUtil.printId(ConnectContext.get().queryId());
ctx.setQueryId(queryId);
- ctx.setWritePath(writeLocation + queryId + "/");
+ ctx.setWritePath(getWritePath());
hmsTransaction.beginInsertTable(ctx);
hmsTransaction.finishInsertTable(new SimpleTableInfo(dbName,
tableName));
hmsTransaction.commit();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
