This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new abe8f22a061 branch-3.0: [enhance](mtmv)Optimize MTMV lock logic #44967
(#45014)
abe8f22a061 is described below
commit abe8f22a0618cc3953a54c3374e05fba5a525c12
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sun Dec 8 10:29:29 2024 -0800
branch-3.0: [enhance](mtmv)Optimize MTMV lock logic #44967 (#45014)
Cherry-picked from #44967
Co-authored-by: zhangdong <[email protected]>
---
.../main/java/org/apache/doris/catalog/Env.java | 46 +++++++++++++---------
.../main/java/org/apache/doris/catalog/MTMV.java | 2 +-
.../java/org/apache/doris/catalog/OlapTable.java | 20 ++++++----
.../org/apache/doris/mtmv/MTMVPartitionUtil.java | 16 +++-----
.../plans/commands/info/ShowCreateMTMVInfo.java | 13 ++----
.../java/org/apache/doris/qe/ShowExecutor.java | 12 ++----
6 files changed, 52 insertions(+), 57 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 61f44a3526b..38dcd3e9a29 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -3414,25 +3414,33 @@ public class Env {
hidePassword, false, specificVersion, false, true);
}
- public static String getMTMVDdl(MTMV mtmv) {
- StringBuilder sb = new StringBuilder("CREATE MATERIALIZED VIEW ");
- sb.append(mtmv.getName());
- addMTMVCols(mtmv, sb);
- sb.append("\n");
- sb.append(mtmv.getRefreshInfo());
- addMTMVKeyInfo(mtmv, sb);
- addTableComment(mtmv, sb);
- addMTMVPartitionInfo(mtmv, sb);
- DistributionInfo distributionInfo = mtmv.getDefaultDistributionInfo();
- sb.append("\n").append(distributionInfo.toSql());
- // properties
- sb.append("\nPROPERTIES (\n");
- addOlapTablePropertyInfo(mtmv, sb, false, false, null);
- addMTMVPropertyInfo(mtmv, sb);
- sb.append("\n)");
- sb.append("\nAS ");
- sb.append(mtmv.getQuerySql());
- return sb.toString();
+ public static String getMTMVDdl(MTMV mtmv) throws AnalysisException {
+ if (!mtmv.tryReadLock(1, TimeUnit.MINUTES)) {
+ throw new AnalysisException(
+ "get table read lock timeout, database=" +
mtmv.getDBName() + ",table=" + mtmv.getName());
+ }
+ try {
+ StringBuilder sb = new StringBuilder("CREATE MATERIALIZED VIEW ");
+ sb.append(mtmv.getName());
+ addMTMVCols(mtmv, sb);
+ sb.append("\n");
+ sb.append(mtmv.getRefreshInfo());
+ addMTMVKeyInfo(mtmv, sb);
+ addTableComment(mtmv, sb);
+ addMTMVPartitionInfo(mtmv, sb);
+ DistributionInfo distributionInfo =
mtmv.getDefaultDistributionInfo();
+ sb.append("\n").append(distributionInfo.toSql());
+ // properties
+ sb.append("\nPROPERTIES (\n");
+ addOlapTablePropertyInfo(mtmv, sb, false, false, null);
+ addMTMVPropertyInfo(mtmv, sb);
+ sb.append("\n)");
+ sb.append("\nAS ");
+ sb.append(mtmv.getQuerySql());
+ return sb.toString();
+ } finally {
+ mtmv.readUnlock();
+ }
}
private static void addMTMVKeyInfo(MTMV mtmv, StringBuilder sb) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
index 955bfd4279f..daf1aac333d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
@@ -392,7 +392,7 @@ public class MTMV extends OlapTable {
Map<String, String> baseToMv = Maps.newHashMap();
Map<PartitionKeyDesc, Set<String>> relatedPartitionDescs =
MTMVPartitionUtil
.generateRelatedPartitionDescs(mvPartitionInfo, mvProperties);
- Map<String, PartitionItem> mvPartitionItems =
getAndCopyPartitionItems();
+ Map<String, PartitionItem> mvPartitionItems =
getAndCopyPartitionItemsWithoutLock();
for (Entry<String, PartitionItem> entry : mvPartitionItems.entrySet())
{
Set<String> basePartitionNames =
relatedPartitionDescs.getOrDefault(entry.getValue().toPartitionKeyDesc(),
Sets.newHashSet());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 543bd49dcbf..b940cc08156 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -3155,19 +3155,23 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
throw new AnalysisException("get table read lock timeout,
database=" + getDBName() + ",table=" + getName());
}
try {
- Map<String, PartitionItem> res = Maps.newHashMap();
- for (Entry<Long, PartitionItem> entry :
getPartitionInfo().getIdToItem(false).entrySet()) {
- Partition partition = idToPartition.get(entry.getKey());
- if (partition != null) {
- res.put(partition.getName(), entry.getValue());
- }
- }
- return res;
+ return getAndCopyPartitionItemsWithoutLock();
} finally {
readUnlock();
}
}
+ public Map<String, PartitionItem> getAndCopyPartitionItemsWithoutLock()
throws AnalysisException {
+ Map<String, PartitionItem> res = Maps.newHashMap();
+ for (Entry<Long, PartitionItem> entry :
getPartitionInfo().getIdToItem(false).entrySet()) {
+ Partition partition = idToPartition.get(entry.getKey());
+ if (partition != null) {
+ res.put(partition.getName(), entry.getValue());
+ }
+ }
+ return res;
+ }
+
@Override
public List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot) {
return getPartitionColumns();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
index 8ba022de415..55ce86e7570 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
@@ -172,18 +172,12 @@ public class MTMVPartitionUtil {
}
public static List<Long> getPartitionsIdsByNames(MTMV mtmv, List<String>
partitions) throws AnalysisException {
- mtmv.readLock();
- try {
- List<Long> res = Lists.newArrayList();
- for (String partitionName : partitions) {
- Partition partition =
mtmv.getPartitionOrAnalysisException(partitionName);
- res.add(partition.getId());
- }
- return res;
- } finally {
- mtmv.readUnlock();
+ List<Long> res = Lists.newArrayList();
+ for (String partitionName : partitions) {
+ Partition partition =
mtmv.getPartitionOrAnalysisException(partitionName);
+ res.add(partition.getId());
}
-
+ return res;
}
/**
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ShowCreateMTMVInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ShowCreateMTMVInfo.java
index 588af236574..b4fde11c3e1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ShowCreateMTMVInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ShowCreateMTMVInfo.java
@@ -74,18 +74,13 @@ public class ShowCreateMTMVInfo {
* @throws DdlException DdlException
* @throws IOException IOException
*/
- public void run(StmtExecutor executor) throws DdlException, IOException {
+ public void run(StmtExecutor executor) throws DdlException, IOException,
org.apache.doris.common.AnalysisException {
List<List<String>> rows = Lists.newArrayList();
Database db =
Env.getCurrentInternalCatalog().getDbOrDdlException(mvName.getDb());
MTMV mtmv = (MTMV) db.getTableOrDdlException(mvName.getTbl());
- mtmv.readLock();
- try {
- String mtmvDdl = Env.getMTMVDdl(mtmv);
- rows.add(Lists.newArrayList(mtmv.getName(), mtmvDdl));
- executor.handleShowCreateMTMVStmt(rows);
- } finally {
- mtmv.readUnlock();
- }
+ String mtmvDdl = Env.getMTMVDdl(mtmv);
+ rows.add(Lists.newArrayList(mtmv.getName(), mtmvDdl));
+ executor.handleShowCreateMTMVStmt(rows);
}
/**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index 9bda42139ef..3dcb7069abf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -1184,15 +1184,9 @@ public class ShowExecutor {
.getDbOrAnalysisException(showStmt.getDb());
MTMV mtmv = (MTMV) db.getTableOrAnalysisException(showStmt.getTable());
List<List<String>> rows = Lists.newArrayList();
-
- mtmv.readLock();
- try {
- String mtmvDdl = Env.getMTMVDdl(mtmv);
- rows.add(Lists.newArrayList(mtmv.getName(), mtmvDdl));
- resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
- } finally {
- mtmv.readUnlock();
- }
+ String mtmvDdl = Env.getMTMVDdl(mtmv);
+ rows.add(Lists.newArrayList(mtmv.getName(), mtmvDdl));
+ resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
}
// Describe statement
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
