This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new eaf1c85512d branch-4.2: [fix](fe) Check automatic partition limit
before creation (#69502)
eaf1c85512d is described below
commit eaf1c85512d5bb96f703e3961f9e2cfe9887df79
Author: Jerry Hu <[email protected]>
AuthorDate: Fri Oct 23 17:36:36 2026 +0800
branch-4.2: [fix](fe) Check automatic partition limit before creation
(#69502)
## Summary
Cherry-picked from #69128 for branch-4.2.
Automatic partition creation previously checked `max_auto_partition_num`
only after creating partitions, so an over-limit request could fail
after leaving new partitions behind. This change counts only missing
partition names under the table read lock and rejects an over-limit
batch before creation. It also rechecks capacity under the table write
lock before publishing each automatic partition, while preserving the
existing behavior of explicit partition DDL.
## Backport notes
The branch-4.2 implementation uses its existing `AddPartitionClause` API
and JUnit 4 test conventions. If concurrent requests exhaust capacity
during a batch, partitions already published within the limit can
remain.
## Tests
- `./build.sh --fe` (passed; FE Checkstyle: 0 violations).
- `./run-fe-ut.sh --run
'org.apache.doris.service.FrontendServiceImplTest#testCreatePartitionRangeLimitDoesNotLeavePartitions+testCreatePartitionListLimitCountsOnlyNewPartitions+testCreatePartitionLimitIsRecheckedAtPublication'`
(3 passed, 0 failures, 0 errors).
- `git diff origin/branch-4.2...HEAD --check` (passed).
---
.../main/java/org/apache/doris/catalog/Env.java | 12 ++-
.../apache/doris/datasource/InternalCatalog.java | 22 +++++
.../apache/doris/service/FrontendServiceImpl.java | 33 ++++++-
.../doris/service/FrontendServiceImplTest.java | 102 +++++++++++++++++++++
4 files changed, 163 insertions(+), 6 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index e73f6c1c0ad..75f92510b71 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -3647,8 +3647,18 @@ public class Env {
boolean writeEditLog,
List<Pair<PartitionPersistInfo,
Partition>> batchPartitions)
throws DdlException {
+ addPartition(db, tableName, addPartitionClause, isCreateTable,
generatedPartitionId,
+ writeEditLog, batchPartitions, false);
+ }
+
+ public void addPartition(Database db, String tableName, AddPartitionClause
addPartitionClause,
+ boolean isCreateTable, long
generatedPartitionId,
+ boolean writeEditLog,
+ List<Pair<PartitionPersistInfo,
Partition>> batchPartitions,
+ boolean enforceAutoPartitionLimit)
+ throws DdlException {
getInternalCatalog().addPartition(db, tableName, addPartitionClause,
- isCreateTable, generatedPartitionId, writeEditLog,
batchPartitions);
+ isCreateTable, generatedPartitionId, writeEditLog,
batchPartitions, enforceAutoPartitionLimit);
}
public void addMultiPartitions(Database db, String tableName,
AlterMultiPartitionClause multiPartitionClause)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 4ebfb9f95f5..adda42ad5c1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -1468,6 +1468,16 @@ public class InternalCatalog implements
CatalogIf<Database> {
boolean writeEditLog,
List<Pair<PartitionPersistInfo,
Partition>> batchPartitions)
throws DdlException {
+ addPartition(db, tableName, addPartitionClause, isCreateTable,
generatedPartitionId,
+ writeEditLog, batchPartitions, false);
+ }
+
+ public void addPartition(Database db, String tableName, AddPartitionClause
addPartitionClause,
+ boolean isCreateTable, long
generatedPartitionId,
+ boolean writeEditLog,
+ List<Pair<PartitionPersistInfo,
Partition>> batchPartitions,
+ boolean enforceAutoPartitionLimit)
+ throws DdlException {
// in cloud mode, isCreateTable == true, create dynamic partition use,
so partitionId must have been generated.
// isCreateTable == false, other case, partitionId generate in below,
must be set 0
if (!FeConstants.runningUnitTest && Config.isCloudMode()
@@ -1747,6 +1757,18 @@ public class InternalCatalog implements
CatalogIf<Database> {
}
}
+ // Check at publication under the table write lock. A request
may have passed the
+ // batch precheck before another request published its own
auto partitions.
+ if (enforceAutoPartitionLimit) {
+ int partitionNum = olapTable.getPartitionNum();
+ int autoPartitionLimit = Config.max_auto_partition_num;
+ if (partitionNum >= autoPartitionLimit) {
+ throw new DdlException(String.format(
+ "partition numbers %d exceeded limit of
variable max_auto_partition_num %d",
+ (long) partitionNum + 1, autoPartitionLimit));
+ }
+ }
+
// check if meta changed
// rollup index may be added or dropped during add partition
operation.
// schema may be changed during add partition operation.
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index f08becb81b0..222276ee773 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -4099,10 +4099,35 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
return result;
}
+ olapTable.readLock();
+ try {
+ int partitionNum = olapTable.getPartitionNum();
+ int newPartitionNum = 0;
+ for (String partitionName : addPartitionClauseMap.keySet()) {
+ if (!olapTable.checkPartitionNameExist(partitionName)) {
+ newPartitionNum++;
+ }
+ }
+ long expectedPartitionNum = (long) partitionNum + newPartitionNum;
+ int autoPartitionLimit = Config.max_auto_partition_num;
+ if (expectedPartitionNum > autoPartitionLimit) {
+ String errorMessage = String.format(
+ "partition numbers %d exceeded limit of variable
max_auto_partition_num %d",
+ expectedPartitionNum, autoPartitionLimit);
+ LOG.warn(errorMessage);
+ errorStatus.setErrorMsgs(Lists.newArrayList(errorMessage));
+ result.setStatus(errorStatus);
+ LOG.warn("send create partition error status: {}", result);
+ return result;
+ }
+ } finally {
+ olapTable.readUnlock();
+ }
+
for (AddPartitionClause addPartitionClause :
addPartitionClauseMap.values()) {
try {
- // here maybe check and limit created partitions num
- Env.getCurrentEnv().addPartition(db, olapTable.getName(),
addPartitionClause, false, 0, true, null);
+ Env.getCurrentEnv().addPartition(db, olapTable.getName(),
addPartitionClause, false, 0, true,
+ null, true);
} catch (DdlException e) {
LOG.warn(e);
errorStatus.setErrorMsgs(
@@ -4128,9 +4153,7 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
olapTable.readLock();
try {
- // check partition's number limit. because partitions in
addPartitionClauseMap may be duplicated with
- // existing partitions, which would lead to false positive. so we
should check the partition number AFTER
- // adding new partitions using its ACTUAL NUMBER, rather than the
sum of existing and requested partitions.
+ // Retain the actual-count check for concurrent changes and
configuration updates during creation.
int partitionNum = olapTable.getPartitionNum();
int autoPartitionLimit = Config.max_auto_partition_num;
if (partitionNum > autoPartitionLimit) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java
b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java
index e45bed95a47..6a4071c08c6 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java
@@ -412,6 +412,108 @@ public class FrontendServiceImplTest {
Assert.assertNotNull(p20230807);
}
+ @Test
+ public void testCreatePartitionRangeLimitDoesNotLeavePartitions() throws
Exception {
+ createTable("CREATE TABLE test.partition_range_limit (event_day
DATETIME NOT NULL, id INT) "
+ + "DUPLICATE KEY(event_day, id) "
+ + "AUTO PARTITION BY RANGE (date_trunc(event_day, 'day')) () "
+ + "DISTRIBUTED BY HASH(id) BUCKETS 1
PROPERTIES('replication_num' = '1')");
+ Database db =
Env.getCurrentInternalCatalog().getDbOrAnalysisException("test");
+ OlapTable table = (OlapTable)
db.getTableOrAnalysisException("partition_range_limit");
+ int oldLimit = Config.max_auto_partition_num;
+ try {
+ Config.max_auto_partition_num = 1;
+ TCreatePartitionResult result = new
FrontendServiceImpl(exeEnv).createPartition(
+ createPartitionRequest(db, table, "2023-08-07 00:00:00",
"2023-08-08 00:00:00"));
+ Assert.assertEquals(TStatusCode.RUNTIME_ERROR,
result.getStatus().getStatusCode());
+
Assert.assertTrue(result.getStatus().getErrorMsgs().get(0).contains("max_auto_partition_num"));
+ Assert.assertEquals(0, table.getPartitionNum());
+ } finally {
+ Config.max_auto_partition_num = oldLimit;
+ }
+ }
+
+ @Test
+ public void testCreatePartitionListLimitCountsOnlyNewPartitions() throws
Exception {
+ createTable("CREATE TABLE test.partition_list_limit (city_code
VARCHAR(100) NOT NULL, id INT) "
+ + "DUPLICATE KEY(city_code, id) "
+ + "AUTO PARTITION BY LIST (city_code) () "
+ + "DISTRIBUTED BY HASH(id) BUCKETS 1
PROPERTIES('replication_num' = '1')");
+ Database db =
Env.getCurrentInternalCatalog().getDbOrAnalysisException("test");
+ OlapTable table = (OlapTable)
db.getTableOrAnalysisException("partition_list_limit");
+ int oldLimit = Config.max_auto_partition_num;
+ try {
+ Config.max_auto_partition_num = 1;
+ FrontendServiceImpl impl = new FrontendServiceImpl(exeEnv);
+ TCreatePartitionResult first =
impl.createPartition(createPartitionRequest(db, table, "BEIJING"));
+ Assert.assertEquals(TStatusCode.OK,
first.getStatus().getStatusCode());
+ Assert.assertEquals(1, table.getPartitionNum());
+
+ TCreatePartitionResult existing =
impl.createPartition(createPartitionRequest(db, table, "BEIJING"));
+ Assert.assertEquals(TStatusCode.OK,
existing.getStatus().getStatusCode());
+ Assert.assertEquals(1, table.getPartitionNum());
+
+ TCreatePartitionResult overflow = impl.createPartition(
+ createPartitionRequest(db, table, "BEIJING", "SHANGHAI"));
+ Assert.assertEquals(TStatusCode.RUNTIME_ERROR,
overflow.getStatus().getStatusCode());
+
Assert.assertTrue(overflow.getStatus().getErrorMsgs().get(0).contains("max_auto_partition_num"));
+ Assert.assertEquals(1, table.getPartitionNum());
+ } finally {
+ Config.max_auto_partition_num = oldLimit;
+ }
+ }
+
+ @Test
+ public void testCreatePartitionLimitIsRecheckedAtPublication() throws
Exception {
+ createTable("CREATE TABLE test.partition_list_concurrent_limit
(city_code VARCHAR(100) NOT NULL, id INT) "
+ + "DUPLICATE KEY(city_code, id) "
+ + "AUTO PARTITION BY LIST (city_code) () "
+ + "DISTRIBUTED BY HASH(id) BUCKETS 1
PROPERTIES('replication_num' = '1')");
+ Database db =
Env.getCurrentInternalCatalog().getDbOrAnalysisException("test");
+ OlapTable table = (OlapTable)
db.getTableOrAnalysisException("partition_list_concurrent_limit");
+ int oldLimit = Config.max_auto_partition_num;
+ try {
+ Config.max_auto_partition_num = 1;
+ FrontendServiceImpl impl = new FrontendServiceImpl(exeEnv);
+ TCreatePartitionResult first =
impl.createPartition(createPartitionRequest(db, table, "BEIJING"));
+ Assert.assertEquals(TStatusCode.OK,
first.getStatus().getStatusCode());
+
+ // Model a concurrent publication after the request's read-lock
precheck.
+ OlapTable spyTable = Mockito.spy(table);
+ AtomicBoolean stalePrecheck = new AtomicBoolean(true);
+ db.unregisterTable(table.getName());
+ db.registerTable(spyTable);
+ try {
+ Mockito.doAnswer(invocation ->
stalePrecheck.compareAndSet(true, false)
+ ? 0 :
invocation.callRealMethod()).when(spyTable).getPartitionNum();
+ TCreatePartitionResult result =
impl.createPartition(createPartitionRequest(db, spyTable, "SHANGHAI"));
+ Assert.assertEquals(TStatusCode.RUNTIME_ERROR,
result.getStatus().getStatusCode());
+
Assert.assertTrue(result.getStatus().getErrorMsgs().get(0).contains("max_auto_partition_num"));
+ Assert.assertFalse(stalePrecheck.get());
+ Assert.assertEquals(1, spyTable.getPartitionNum());
+ } finally {
+ db.unregisterTable(spyTable.getName());
+ db.registerTable(table);
+ }
+ } finally {
+ Config.max_auto_partition_num = oldLimit;
+ }
+ }
+
+ private static TCreatePartitionRequest createPartitionRequest(Database db,
OlapTable table, String... values) {
+ List<List<TNullableStringLiteral>> partitionValues = new ArrayList<>();
+ for (String value : values) {
+ TNullableStringLiteral literal = new TNullableStringLiteral();
+ literal.setValue(value);
+ partitionValues.add(Collections.singletonList(literal));
+ }
+ TCreatePartitionRequest request = new TCreatePartitionRequest();
+ request.setDbId(db.getId());
+ request.setTableId(table.getId());
+ request.setPartitionValues(partitionValues);
+ return request;
+ }
+
@Test
public void
testCreatePartitionReturnsRetryErrorWhenResultPartitionIsMissing() throws
Exception {
String createOlapTblStmt = "CREATE TABLE
test.partition_dropped_before_result_snapshot(\n"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
