This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 11dbf5ad756 [opt](mtmv) generate bi-map between base table and
materialized view partitions (#35131)
11dbf5ad756 is described below
commit 11dbf5ad756761b94bc01b19059b5f7e9f071cda
Author: zhangdong <[email protected]>
AuthorDate: Wed May 22 20:07:06 2024 +0800
[opt](mtmv) generate bi-map between base table and materialized view
partitions (#35131)
---
.../main/java/org/apache/doris/catalog/MTMV.java | 39 ++++++++++++++
.../test/java/org/apache/doris/mtmv/MTMVTest.java | 61 ++++++++++++++++++++++
2 files changed, 100 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
index d95c2388a28..1b4a8e7063e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
@@ -20,6 +20,7 @@ package org.apache.doris.catalog;
import org.apache.doris.analysis.PartitionKeyDesc;
import org.apache.doris.catalog.OlapTableFactory.MTMVParams;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Pair;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.job.common.TaskStatus;
@@ -316,6 +317,44 @@ public class MTMV extends OlapTable {
return result;
}
+ /**
+ * Calculate the partition and associated partition mapping relationship
of the MTMV
+ * It is the result of real-time comparison calculation, so there may be
some costs,
+ * so it should be called with caution.
+ * The reason for not directly calling `calculatePartitionMappings` and
+ * generating a reverse index is to directly generate two maps here,
+ * without the need to traverse them again
+ *
+ * @return mvPartitionName ==> relationPartitionNames and
relationPartitionName ==> mvPartitionName
+ * @throws AnalysisException
+ */
+ public Pair<Map<String, Set<String>>, Map<String, String>>
calculateDoublyPartitionMappings()
+ throws AnalysisException {
+ if (mvPartitionInfo.getPartitionType() ==
MTMVPartitionType.SELF_MANAGE) {
+ return Pair.of(Maps.newHashMap(), Maps.newHashMap());
+ }
+ long start = System.currentTimeMillis();
+ Map<String, Set<String>> mvToBase = Maps.newHashMap();
+ Map<String, String> baseToMv = Maps.newHashMap();
+ Map<PartitionKeyDesc, Set<String>> relatedPartitionDescs =
MTMVPartitionUtil
+ .generateRelatedPartitionDescs(mvPartitionInfo, mvProperties);
+ Map<String, PartitionItem> mvPartitionItems =
getAndCopyPartitionItems();
+ for (Entry<String, PartitionItem> entry : mvPartitionItems.entrySet())
{
+ Set<String> basePartitionNames =
relatedPartitionDescs.getOrDefault(entry.getValue().toPartitionKeyDesc(),
+ Sets.newHashSet());
+ String mvPartitionName = entry.getKey();
+ mvToBase.put(mvPartitionName, basePartitionNames);
+ for (String basePartitionName : basePartitionNames) {
+ baseToMv.put(basePartitionName, mvPartitionName);
+ }
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("calculateDoublyPartitionMappings use [{}] mills, mvName
is [{}]",
+ System.currentTimeMillis() - start, name);
+ }
+ return Pair.of(mvToBase, baseToMv);
+ }
+
/**
* Calculate the partition and associated partition mapping relationship
of the MTMV
* It is the result of real-time comparison calculation, so there may be
some costs,
diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java
b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java
index 4a3bf12f889..f004124d50d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java
@@ -17,18 +17,33 @@
package org.apache.doris.mtmv;
+import org.apache.doris.analysis.PartitionKeyDesc;
+import org.apache.doris.analysis.PartitionValue;
+import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.MTMV;
+import org.apache.doris.catalog.PartitionItem;
+import org.apache.doris.catalog.PartitionKey;
+import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.catalog.RangePartitionItem;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.AnalysisException;
import org.apache.doris.job.common.IntervalUnit;
import org.apache.doris.job.extensions.mtmv.MTMVTask;
import org.apache.doris.mtmv.MTMVRefreshEnum.BuildMode;
import org.apache.doris.mtmv.MTMVRefreshEnum.RefreshMethod;
import org.apache.doris.mtmv.MTMVRefreshEnum.RefreshTrigger;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Range;
import com.google.common.collect.Sets;
import org.junit.Assert;
import org.junit.Test;
import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
public class MTMVTest {
@Test
@@ -83,4 +98,50 @@ public class MTMVTest {
task.setTaskId(1L);
return task;
}
+
+ @Test
+ public void testCalculateDoublyPartitionMappings() throws
AnalysisException {
+ Map<String, Set<String>> mvToBase = Maps.newHashMap();
+ Map<String, String> baseToMv = Maps.newHashMap();
+ Map<PartitionKeyDesc, Set<String>> relatedPartitionDescs =
mockRelatedPartitionDescs();
+ Map<String, PartitionItem> mvPartitionItems = mockMvPartitionItems();
+ for (Entry<String, PartitionItem> entry : mvPartitionItems.entrySet())
{
+ Set<String> basePartitionNames =
relatedPartitionDescs.getOrDefault(entry.getValue().toPartitionKeyDesc(),
+ Sets.newHashSet());
+ String mvPartitionName = entry.getKey();
+ mvToBase.put(mvPartitionName, basePartitionNames);
+ for (String basePartitionName : basePartitionNames) {
+ baseToMv.put(basePartitionName, mvPartitionName);
+ }
+ }
+ Assert.assertEquals(mvToBase.get("mvp1"), Sets.newHashSet("baseP1_1",
"baseP1_2"));
+ Assert.assertEquals(baseToMv.get("baseP1_1"), "mvp1");
+ Assert.assertEquals(baseToMv.get("baseP1_2"), "mvp1");
+ }
+
+ private Map<PartitionKeyDesc, Set<String>> mockRelatedPartitionDescs()
throws AnalysisException {
+ Map<PartitionKeyDesc, Set<String>> res = Maps.newHashMap();
+ Column k1 = new Column("k1",
ScalarType.createType(PrimitiveType.TINYINT), true, null, "", "key1");
+ PartitionKey rangeP1Lower =
PartitionKey.createPartitionKey(Lists.newArrayList(new PartitionValue("1")),
+ Lists.newArrayList(k1));
+ PartitionKey rangeP1Upper =
PartitionKey.createPartitionKey(Lists.newArrayList(new PartitionValue("10")),
+ Lists.newArrayList(k1));
+ Range<PartitionKey> rangeP1 = Range.closedOpen(rangeP1Lower,
rangeP1Upper);
+ PartitionItem item1 = new RangePartitionItem(rangeP1);
+ res.put(item1.toPartitionKeyDesc(), Sets.newHashSet("baseP1_1",
"baseP1_2"));
+ return res;
+ }
+
+ private Map<String, PartitionItem> mockMvPartitionItems() throws
AnalysisException {
+ Map<String, PartitionItem> res = Maps.newHashMap();
+ Column k1 = new Column("k1",
ScalarType.createType(PrimitiveType.TINYINT), true, null, "", "key1");
+ PartitionKey rangeP1Lower =
PartitionKey.createPartitionKey(Lists.newArrayList(new PartitionValue("1")),
+ Lists.newArrayList(k1));
+ PartitionKey rangeP1Upper =
PartitionKey.createPartitionKey(Lists.newArrayList(new PartitionValue("10")),
+ Lists.newArrayList(k1));
+ Range<PartitionKey> rangeP1 = Range.closedOpen(rangeP1Lower,
rangeP1Upper);
+ PartitionItem item1 = new RangePartitionItem(rangeP1);
+ res.put("mvp1", item1);
+ return res;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
