github-actions[bot] commented on code in PR #64878:
URL: https://github.com/apache/doris/pull/64878#discussion_r3602342369
##########
fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java:
##########
@@ -766,91 +788,126 @@ public Map<String, String> getCustomProperties() {
return getMaskedCustomProperties("property.");
}
+ @Override
+ public boolean appliesRoutineLoadDescAtomically() {
+ return true;
+ }
+
@Override
public void modifyProperties(AlterRoutineLoadCommand command) throws
UserException {
Map<String, String> jobProperties = command.getAnalyzedJobProperties();
KafkaDataSourceProperties dataSourceProperties =
(KafkaDataSourceProperties) command.getDataSourceProperties();
- if (null != dataSourceProperties) {
- // if the partition offset is set by timestamp, convert it to real
offset
- convertOffset(dataSourceProperties);
- }
writeLock();
try {
if (getState() != JobState.PAUSED) {
throw new DdlException("Only supports modification of PAUSED
jobs");
}
+ if (command.getRoutineLoadDesc() != null &&
command.getLoadPropertyTableId() != tableId) {
+ throw new DdlException("Routine load target table changed
while validating load properties; "
+ + "please retry ALTER ROUTINE LOAD");
+ }
+ if (!command.hasTargetTable()) {
+ validateAlterJobPropertiesForMutation(command);
+ }
- modifyPropertiesInternal(jobProperties, dataSourceProperties);
+ // Kafka offset resolution may perform external I/O. Keep it
outside DB/table metadata locks.
+ PreparedKafkaProperties preparedDataSourceProperties =
+ prepareDataSourceProperties(dataSourceProperties, false);
+ if (!command.hasTargetTable()) {
+ unprotectApplyAlter(command, dataSourceProperties,
preparedDataSourceProperties);
+ return;
+ }
- AlterRoutineLoadJobOperationLog log = new
AlterRoutineLoadJobOperationLog(this.id,
- jobProperties, dataSourceProperties);
- Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log);
+ Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId);
+ if (db == null) {
+ throw new DdlException("Database not found: " + dbId);
+ }
+ db.readLock();
+ try {
+ Table table = db.getTableNullable(command.getTargetTableId());
+ if (table == null) {
+ throw new DdlException("Table not found: " +
command.getTargetTableId());
+ }
+ if (!(table instanceof OlapTable)) {
+ throw new DdlException("Routine load target table must be
an OLAP table");
+ }
+ table.readLock();
+ try {
+ TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode =
getEffectiveUniqueKeyUpdateMode(
+ uniqueKeyUpdateMode, jobProperties);
+ unprotectedValidateTargetTable(
+ db, (OlapTable) table, jobProperties,
effectiveUniqueKeyUpdateMode);
+ unprotectApplyAlter(command, dataSourceProperties,
preparedDataSourceProperties);
+ } finally {
+ table.readUnlock();
+ }
+ } finally {
+ db.readUnlock();
+ }
} finally {
writeUnlock();
}
}
- private void convertOffset(KafkaDataSourceProperties dataSourceProperties)
throws UserException {
+ private void unprotectApplyAlter(AlterRoutineLoadCommand command,
+ KafkaDataSourceProperties dataSourceProperties,
+ PreparedKafkaProperties preparedDataSourceProperties) throws
UserException {
+ modifyPropertiesInternal(command.getAnalyzedJobProperties(),
dataSourceProperties,
+ preparedDataSourceProperties);
+ if (command.hasTargetTable()) {
+ tableId = command.getTargetTableId();
Review Comment:
[P1] Make target/property alters survive checkpoint reload. EditLog replay
restores this target and the new job properties, but
RoutineLoadJob.gsonPostProcess() later reparses the unchanged CREATE origStmt,
substitutes only the current table name, and validates the original properties
against it. For example, a job created in fixed partial-update mode on MoW
table A can switch to non-MoW B together with unique_key_update_mode=UPSERT,
then be marked CANCELLED when an image reload validates the old fixed mode
against B. Preserve the serialized current definition (or update the canonical
statement) and add a full RoutineLoadJob image round-trip test.
##########
fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java:
##########
@@ -766,91 +788,126 @@ public Map<String, String> getCustomProperties() {
return getMaskedCustomProperties("property.");
}
+ @Override
+ public boolean appliesRoutineLoadDescAtomically() {
+ return true;
+ }
+
@Override
public void modifyProperties(AlterRoutineLoadCommand command) throws
UserException {
Map<String, String> jobProperties = command.getAnalyzedJobProperties();
KafkaDataSourceProperties dataSourceProperties =
(KafkaDataSourceProperties) command.getDataSourceProperties();
- if (null != dataSourceProperties) {
- // if the partition offset is set by timestamp, convert it to real
offset
- convertOffset(dataSourceProperties);
- }
writeLock();
try {
if (getState() != JobState.PAUSED) {
throw new DdlException("Only supports modification of PAUSED
jobs");
}
+ if (command.getRoutineLoadDesc() != null &&
command.getLoadPropertyTableId() != tableId) {
+ throw new DdlException("Routine load target table changed
while validating load properties; "
+ + "please retry ALTER ROUTINE LOAD");
+ }
+ if (!command.hasTargetTable()) {
+ validateAlterJobPropertiesForMutation(command);
+ }
- modifyPropertiesInternal(jobProperties, dataSourceProperties);
+ // Kafka offset resolution may perform external I/O. Keep it
outside DB/table metadata locks.
+ PreparedKafkaProperties preparedDataSourceProperties =
+ prepareDataSourceProperties(dataSourceProperties, false);
+ if (!command.hasTargetTable()) {
+ unprotectApplyAlter(command, dataSourceProperties,
preparedDataSourceProperties);
+ return;
+ }
- AlterRoutineLoadJobOperationLog log = new
AlterRoutineLoadJobOperationLog(this.id,
- jobProperties, dataSourceProperties);
- Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log);
+ Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId);
+ if (db == null) {
+ throw new DdlException("Database not found: " + dbId);
+ }
+ db.readLock();
+ try {
+ Table table = db.getTableNullable(command.getTargetTableId());
+ if (table == null) {
+ throw new DdlException("Table not found: " +
command.getTargetTableId());
+ }
+ if (!(table instanceof OlapTable)) {
+ throw new DdlException("Routine load target table must be
an OLAP table");
+ }
+ table.readLock();
+ try {
+ TUniqueKeyUpdateMode effectiveUniqueKeyUpdateMode =
getEffectiveUniqueKeyUpdateMode(
+ uniqueKeyUpdateMode, jobProperties);
+ unprotectedValidateTargetTable(
+ db, (OlapTable) table, jobProperties,
effectiveUniqueKeyUpdateMode);
+ unprotectApplyAlter(command, dataSourceProperties,
preparedDataSourceProperties);
+ } finally {
+ table.readUnlock();
+ }
+ } finally {
+ db.readUnlock();
+ }
} finally {
writeUnlock();
}
}
- private void convertOffset(KafkaDataSourceProperties dataSourceProperties)
throws UserException {
+ private void unprotectApplyAlter(AlterRoutineLoadCommand command,
+ KafkaDataSourceProperties dataSourceProperties,
+ PreparedKafkaProperties preparedDataSourceProperties) throws
UserException {
+ modifyPropertiesInternal(command.getAnalyzedJobProperties(),
dataSourceProperties,
+ preparedDataSourceProperties);
+ if (command.hasTargetTable()) {
+ tableId = command.getTargetTableId();
+ }
+ setRoutineLoadDesc(command.getRoutineLoadDesc());
+ AlterRoutineLoadJobOperationLog log = new
AlterRoutineLoadJobOperationLog(id,
+ command.getAnalyzedJobProperties(), dataSourceProperties,
command.getTargetTableId(),
+ command.getRoutineLoadDesc());
+ Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log);
Review Comment:
[P1] Do not wait for EditLog durability while holding the database and table
metadata locks. This target path reaches synchronous logEdit with the job
write, database read, and target-table read locks held. With batch logging,
queue insertion retries indefinitely and logEditWithQueue waits without a
timeout, so journal backpressure can indefinitely block DDL needing either
metadata write lock; direct logging is synchronous too. This remains after
Kafka/MetaService I/O was moved out of the locked region. Restructure the
commit so journal ordering is preserved but no blocking journal operation runs
under the metadata locks; the existing submitEdit/await split can separate
completion in batch mode, while its synchronous direct-mode fallback also needs
to stay outside those locks.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
