This is an automated email from the ASF dual-hosted git repository.
caiconghui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3070318f95 [Enhancement](IdGenerator) Use IdGeneratorBuffer to get
better performance for creating tablet in fe when do alter table job (#11524)
3070318f95 is described below
commit 3070318f95ad486466be3021019d9b7eeebcd1ad
Author: caiconghui <[email protected]>
AuthorDate: Fri Aug 5 23:27:29 2022 +0800
[Enhancement](IdGenerator) Use IdGeneratorBuffer to get better performance
for creating tablet in fe when do alter table job (#11524)
Co-authored-by: caiconghui1 <[email protected]>
---
.../doris/alter/MaterializedViewHandler.java | 13 +++++--
.../apache/doris/alter/SchemaChangeHandler.java | 44 ++++++++++++----------
.../org/apache/doris/catalog/MetaIdGenerator.java | 7 +++-
.../apache/doris/common/util/IdGeneratorUtil.java | 24 ++++++++++--
.../doris/datasource/InternalDataSource.java | 4 +-
.../apache/doris/catalog/MetaIdGeneratorTest.java | 44 ++++++++++++++++++++++
6 files changed, 105 insertions(+), 31 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
index 568b3623b5..504db3aa71 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
@@ -32,6 +32,7 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.MaterializedIndex.IndexState;
+import org.apache.doris.catalog.MetaIdGenerator.IdGeneratorBuffer;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.OlapTable.OlapTableState;
import org.apache.doris.catalog.Partition;
@@ -47,6 +48,7 @@ import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.MetaNotFoundException;
+import org.apache.doris.common.util.IdGeneratorUtil;
import org.apache.doris.common.util.ListComparator;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.common.util.Util;
@@ -63,6 +65,7 @@ import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -343,8 +346,10 @@ public class MaterializedViewHandler extends AlterHandler {
long tableId = olapTable.getId();
int baseSchemaHash = olapTable.getSchemaHashByIndexId(baseIndexId);
Env env = Env.getCurrentEnv();
- long jobId = env.getNextId();
- long mvIndexId = env.getNextId();
+ long bufferSize =
IdGeneratorUtil.getBufferSizeForAlterTable(olapTable,
Sets.newHashSet(baseIndexId));
+ IdGeneratorBuffer idGeneratorBuffer =
env.getIdGeneratorBuffer(bufferSize);
+ long jobId = idGeneratorBuffer.getNextId();
+ long mvIndexId = idGeneratorBuffer.getNextId();
RollupJobV2 mvJob = new RollupJobV2(jobId, dbId, tableId,
olapTable.getName(), timeoutMs,
baseIndexId, mvIndexId, baseIndexName, mvName,
mvColumns, baseSchemaHash, mvSchemaHash,
@@ -372,7 +377,7 @@ public class MaterializedViewHandler extends AlterHandler {
short replicationNum =
olapTable.getPartitionInfo().getReplicaAllocation(partitionId).getTotalReplicaNum();
for (Tablet baseTablet : baseIndex.getTablets()) {
long baseTabletId = baseTablet.getId();
- long mvTabletId = env.getNextId();
+ long mvTabletId = idGeneratorBuffer.getNextId();
Tablet newTablet = new Tablet(mvTabletId);
mvIndex.addTablet(newTablet, mvTabletMeta);
@@ -383,7 +388,7 @@ public class MaterializedViewHandler extends AlterHandler {
int healthyReplicaNum = 0;
for (Replica baseReplica : baseReplicas) {
- long mvReplicaId = env.getNextId();
+ long mvReplicaId = idGeneratorBuffer.getNextId();
long backendId = baseReplica.getBackendId();
if (baseReplica.getState() == ReplicaState.CLONE
|| baseReplica.getState() ==
ReplicaState.DECOMMISSION
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index be4fc53cea..e2c3d0f8b3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -43,6 +43,7 @@ import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.MaterializedIndex.IndexExtState;
import org.apache.doris.catalog.MaterializedIndex.IndexState;
import org.apache.doris.catalog.MaterializedIndexMeta;
+import org.apache.doris.catalog.MetaIdGenerator.IdGeneratorBuffer;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.OlapTable.OlapTableState;
import org.apache.doris.catalog.Partition;
@@ -66,6 +67,7 @@ import org.apache.doris.common.Pair;
import org.apache.doris.common.ThreadPoolManager;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.DynamicPartitionUtil;
+import org.apache.doris.common.util.IdGeneratorUtil;
import org.apache.doris.common.util.ListComparator;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.common.util.Util;
@@ -1218,23 +1220,6 @@ public class SchemaChangeHandler extends AlterHandler {
TStorageFormat storageFormat =
PropertyAnalyzer.analyzeStorageFormat(propertyMap);
- // create job
- Env env = Env.getCurrentEnv();
- long jobId = env.getNextId();
- SchemaChangeJobV2 schemaChangeJob = new SchemaChangeJobV2(jobId, dbId,
- olapTable.getId(), olapTable.getName(), timeoutSecond * 1000);
- schemaChangeJob.setBloomFilterInfo(hasBfChange, bfColumns, bfFpp);
- schemaChangeJob.setAlterIndexInfo(hasIndexChange, indexes);
-
- // If StorageFormat is set to TStorageFormat.V2
- // which will create tablet with preferred_rowset_type set to BETA
- // for both base table and rollup index
- if (hasIndexChange) {
- // only V2 support index, so if there is index changed, storage
format must be V2
- storageFormat = TStorageFormat.V2;
- }
- schemaChangeJob.setStorageFormat(storageFormat);
-
// begin checking each table
// ATTN: DO NOT change any meta in this loop
long tableId = olapTable.getId();
@@ -1413,6 +1398,24 @@ public class SchemaChangeHandler extends AlterHandler {
throw new DdlException("Nothing is changed. please check your
alter stmt.");
}
+ // create job
+ long bufferSize =
IdGeneratorUtil.getBufferSizeForAlterTable(olapTable,
changedIndexIdToSchema.keySet());
+ IdGeneratorBuffer idGeneratorBuffer =
Env.getCurrentEnv().getIdGeneratorBuffer(bufferSize);
+ long jobId = idGeneratorBuffer.getNextId();
+ SchemaChangeJobV2 schemaChangeJob = new SchemaChangeJobV2(jobId, dbId,
+ olapTable.getId(), olapTable.getName(), timeoutSecond * 1000);
+ schemaChangeJob.setBloomFilterInfo(hasBfChange, bfColumns, bfFpp);
+ schemaChangeJob.setAlterIndexInfo(hasIndexChange, indexes);
+
+ // If StorageFormat is set to TStorageFormat.V2
+ // which will create tablet with preferred_rowset_type set to BETA
+ // for both base table and rollup index
+ if (hasIndexChange) {
+ // only V2 support index, so if there is index changed, storage
format must be V2
+ storageFormat = TStorageFormat.V2;
+ }
+ schemaChangeJob.setStorageFormat(storageFormat);
+
// the following operations are done outside the 'for indices' loop
// to avoid partial check success
@@ -1437,7 +1440,7 @@ public class SchemaChangeHandler extends AlterHandler {
}
String newIndexName = SHADOW_NAME_PRFIX +
olapTable.getIndexNameById(originIndexId);
short newShortKeyColumnCount =
indexIdToShortKeyColumnCount.get(originIndexId);
- long shadowIndexId = env.getNextId();
+ long shadowIndexId = idGeneratorBuffer.getNextId();
// create SHADOW index for each partition
List<Tablet> addedTablets = Lists.newArrayList();
@@ -1453,7 +1456,7 @@ public class SchemaChangeHandler extends AlterHandler {
Short totalReplicaNum = replicaAlloc.getTotalReplicaNum();
for (Tablet originTablet : originIndex.getTablets()) {
long originTabletId = originTablet.getId();
- long shadowTabletId = env.getNextId();
+ long shadowTabletId = idGeneratorBuffer.getNextId();
Tablet shadowTablet = new Tablet(shadowTabletId);
shadowIndex.addTablet(shadowTablet, shadowTabletMeta);
@@ -1464,11 +1467,12 @@ public class SchemaChangeHandler extends AlterHandler {
int healthyReplicaNum = 0;
for (Replica originReplica : originReplicas) {
- long shadowReplicaId = env.getNextId();
+ long shadowReplicaId = idGeneratorBuffer.getNextId();
long backendId = originReplica.getBackendId();
if (originReplica.getState() ==
Replica.ReplicaState.CLONE
|| originReplica.getState() ==
Replica.ReplicaState.DECOMMISSION
+ || originReplica.getState() ==
ReplicaState.COMPACTION_TOO_SLOW
|| originReplica.getLastFailedVersion() > 0) {
LOG.info("origin replica {} of tablet {} state is
{},"
+ " and last failed version is {},
skip creating shadow replica",
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/MetaIdGenerator.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MetaIdGenerator.java
index d2f16a6fc2..65e30320c2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MetaIdGenerator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MetaIdGenerator.java
@@ -58,7 +58,8 @@ public class MetaIdGenerator {
IdGeneratorBuffer idGeneratorBuffer = new IdGeneratorBuffer(nextId,
nextId + bufferSize - 1);
nextId = nextId + bufferSize;
if (nextId > batchEndId) {
- batchEndId = batchEndId + (bufferSize / BATCH_ID_INTERVAL + 1) *
BATCH_ID_INTERVAL;
+ batchEndId = batchEndId + ((nextId - batchEndId) /
BATCH_ID_INTERVAL + 1) * BATCH_ID_INTERVAL;
+ Preconditions.checkState(nextId <= batchEndId);
if (editLog != null) {
editLog.logSaveNextId(batchEndId);
}
@@ -91,5 +92,9 @@ public class MetaIdGenerator {
Preconditions.checkState(nextId <= batchEndId);
return nextId++;
}
+
+ public long getBatchEndId() {
+ return batchEndId;
+ }
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/IdGeneratorUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/IdGeneratorUtil.java
index b9609d409a..1369c0b405 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/IdGeneratorUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/IdGeneratorUtil.java
@@ -19,17 +19,20 @@ package org.apache.doris.common.util;
import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.analysis.SinglePartitionDesc;
+import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.ReplicaAllocation;
-import org.apache.doris.common.AnalysisException;
+import org.apache.doris.catalog.Tablet;
import org.apache.doris.common.DdlException;
import java.util.Collection;
+import java.util.Set;
public class IdGeneratorUtil {
- public static long getBufferSize(CreateTableStmt stmt, ReplicaAllocation
replicaAlloc) throws DdlException,
- AnalysisException {
+ public static long getBufferSizeForCreateTable(CreateTableStmt stmt,
ReplicaAllocation replicaAlloc)
+ throws DdlException {
long bufferSize = 1;
long partitionNum = stmt.getPartitionDesc() == null ? 1 :
stmt.getPartitionDesc().getSinglePartitionDescs().size();
@@ -47,7 +50,7 @@ public class IdGeneratorUtil {
return bufferSize;
}
- public static long getBufferSize(OlapTable table, Collection<Long>
partitionIds) {
+ public static long getBufferSizeForTruncateTable(OlapTable table,
Collection<Long> partitionIds) {
long bufferSize = 0;
for (Long partitionId : partitionIds) {
bufferSize = bufferSize + 1;
@@ -58,4 +61,17 @@ public class IdGeneratorUtil {
}
return bufferSize;
}
+
+ public static long getBufferSizeForAlterTable(OlapTable table, Set<Long>
indexIdSet) {
+ long bufferSize = 1 + indexIdSet.size();
+ for (Long indexId : indexIdSet) {
+ for (Partition partition : table.getPartitions()) {
+ MaterializedIndex originIndex = partition.getIndex(indexId);
+ for (Tablet baseTablet : originIndex.getTablets()) {
+ bufferSize = bufferSize + 1 +
baseTablet.getReplicas().size();
+ }
+ }
+ }
+ return bufferSize;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalDataSource.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalDataSource.java
index a490745d74..27ceedcc8f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalDataSource.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalDataSource.java
@@ -1670,7 +1670,7 @@ public class InternalDataSource implements
DataSourceIf<Database> {
replicaAlloc = ReplicaAllocation.DEFAULT_ALLOCATION;
}
- long bufferSize = IdGeneratorUtil.getBufferSize(stmt, replicaAlloc);
+ long bufferSize = IdGeneratorUtil.getBufferSizeForCreateTable(stmt,
replicaAlloc);
IdGeneratorBuffer idGeneratorBuffer =
Env.getCurrentEnv().getIdGeneratorBuffer(bufferSize);
// create partition info
@@ -2376,7 +2376,7 @@ public class InternalDataSource implements
DataSourceIf<Database> {
List<Partition> newPartitions = Lists.newArrayList();
// tabletIdSet to save all newly created tablet ids.
Set<Long> tabletIdSet = Sets.newHashSet();
- long bufferSize = IdGeneratorUtil.getBufferSize(copiedTbl,
origPartitions.values());
+ long bufferSize =
IdGeneratorUtil.getBufferSizeForTruncateTable(copiedTbl,
origPartitions.values());
IdGeneratorBuffer idGeneratorBuffer =
Env.getCurrentEnv().getIdGeneratorBuffer(bufferSize);
try {
for (Map.Entry<String, Long> entry : origPartitions.entrySet()) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/MetaIdGeneratorTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/MetaIdGeneratorTest.java
new file mode 100644
index 0000000000..7bd1b34158
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/MetaIdGeneratorTest.java
@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.catalog;
+
+import org.apache.doris.catalog.MetaIdGenerator.IdGeneratorBuffer;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+
+public class MetaIdGeneratorTest {
+
+ @Test
+ public void normalTest() {
+ MetaIdGenerator idGenerator = new MetaIdGenerator(10);
+ Assert.assertEquals(10, idGenerator.getBatchEndId());
+ Assert.assertEquals(11, idGenerator.getNextId());
+ Assert.assertEquals(1010, idGenerator.getBatchEndId());
+
+ IdGeneratorBuffer idGeneratorBuffer =
idGenerator.getIdGeneratorBuffer(3500);
+ Assert.assertEquals(12, idGeneratorBuffer.getNextId());
+ Assert.assertEquals(4010, idGenerator.getBatchEndId());
+ for (int i = 1; i < 3500; i++) {
+ Assert.assertEquals(i + 12, idGeneratorBuffer.getNextId());
+ }
+ Assert.assertEquals(3511, idGeneratorBuffer.getBatchEndId());
+ Assert.assertEquals(3512, idGenerator.getNextId());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
