This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 2a6d6c15c5a [enhance](hive)hive event code optimization #42637 (#42876)
2a6d6c15c5a is described below
commit 2a6d6c15c5a1d948c9a8751559b95db3551c6a25
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Wed Oct 30 12:52:21 2024 +0800
[enhance](hive)hive event code optimization #42637 (#42876)
cherry pick from #42637
Co-authored-by: zhangdong <[email protected]>
---
.../datasource/hive/event/AddPartitionEvent.java | 8 +-
.../datasource/hive/event/AlterDatabaseEvent.java | 8 +-
.../datasource/hive/event/AlterPartitionEvent.java | 6 +-
.../datasource/hive/event/AlterTableEvent.java | 10 +-
.../datasource/hive/event/CreateDatabaseEvent.java | 4 +-
.../datasource/hive/event/CreateTableEvent.java | 8 +-
.../datasource/hive/event/DropDatabaseEvent.java | 4 +-
.../datasource/hive/event/DropPartitionEvent.java | 8 +-
.../datasource/hive/event/DropTableEvent.java | 6 +-
.../doris/datasource/hive/event/IgnoredEvent.java | 2 +-
.../doris/datasource/hive/event/InsertEvent.java | 6 +-
.../datasource/hive/event/MetastoreEvent.java | 167 ++++++---------------
.../hive/event/MetastoreEventsProcessor.java | 2 +-
13 files changed, 81 insertions(+), 158 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java
index ffc7b95ff59..e582c3d2662 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java
@@ -59,7 +59,7 @@ public class AddPartitionEvent extends
MetastorePartitionEvent {
super(event, catalogName);
Preconditions.checkArgument(getEventType().equals(MetastoreEventType.ADD_PARTITION));
Preconditions
- .checkNotNull(event.getMessage(), debugString("Event message
is null"));
+ .checkNotNull(event.getMessage(), getMsgWithEventInfo("Event
message is null"));
try {
AddPartitionMessage addPartitionMessage =
MetastoreEventsProcessor.getMessageDeserializer(event.getMessageFormat())
@@ -98,18 +98,18 @@ public class AddPartitionEvent extends
MetastorePartitionEvent {
@Override
protected void process() throws MetastoreNotificationException {
try {
-
infoLog("catalogName:[{}],dbName:[{}],tableName:[{}],partitionNames:[{}]",
catalogName, dbName, tblName,
+
logInfo("catalogName:[{}],dbName:[{}],tableName:[{}],partitionNames:[{}]",
catalogName, dbName, tblName,
partitionNames.toString());
// bail out early if there are not partitions to process
if (partitionNames.isEmpty()) {
- infoLog("Partition list is empty. Ignoring this event.");
+ logInfo("Partition list is empty. Ignoring this event.");
return;
}
Env.getCurrentEnv().getCatalogMgr()
.addExternalPartitions(catalogName, dbName,
hmsTbl.getTableName(), partitionNames, eventTime, true);
} catch (DdlException e) {
throw new MetastoreNotificationException(
- debugString("Failed to process event"), e);
+ getMsgWithEventInfo("Failed to process event"), e);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java
index ef828025e7f..334930861b6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java
@@ -67,7 +67,7 @@ public class AlterDatabaseEvent extends MetastoreEvent {
dbNameAfter = dbAfter.getName();
} catch (Exception e) {
throw new MetastoreNotificationException(
- debugString("Unable to parse the alter database message"),
e);
+ getMsgWithEventInfo("Unable to parse the alter database
message"), e);
}
// this is a rename event if either dbName of before and after object
changed
isRename = !dbBefore.getName().equalsIgnoreCase(dbAfter.getName());
@@ -82,7 +82,7 @@ public class AlterDatabaseEvent extends MetastoreEvent {
throw new DdlException("Only support ExternalCatalog Databases");
}
if (catalog.getDbNullable(dbAfter.getName()) != null) {
- infoLog("AlterExternalDatabase canceled, because dbAfter has
exist, "
+ logInfo("AlterExternalDatabase canceled, because dbAfter has
exist, "
+ "catalogName:[{}],dbName:[{}]",
catalogName, dbAfter.getName());
return;
@@ -113,10 +113,10 @@ public class AlterDatabaseEvent extends MetastoreEvent {
return;
}
// only can change properties,we do nothing
- infoLog("catalogName:[{}],dbName:[{}]", catalogName, dbName);
+ logInfo("catalogName:[{}],dbName:[{}]", catalogName, dbName);
} catch (Exception e) {
throw new MetastoreNotificationException(
- debugString("Failed to process event"), e);
+ getMsgWithEventInfo("Failed to process event"), e);
}
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java
index 569d9878d7a..d9898f68d98 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java
@@ -65,7 +65,7 @@ public class AlterPartitionEvent extends
MetastorePartitionEvent {
super(event, catalogName);
Preconditions.checkArgument(getEventType().equals(MetastoreEventType.ALTER_PARTITION));
Preconditions
- .checkNotNull(event.getMessage(), debugString("Event message
is null"));
+ .checkNotNull(event.getMessage(), getMsgWithEventInfo("Event
message is null"));
try {
AlterPartitionMessage alterPartitionMessage =
MetastoreEventsProcessor.getMessageDeserializer(event.getMessageFormat())
@@ -109,7 +109,7 @@ public class AlterPartitionEvent extends
MetastorePartitionEvent {
@Override
protected void process() throws MetastoreNotificationException {
try {
-
infoLog("catalogName:[{}],dbName:[{}],tableName:[{}],partitionNameBefore:[{}],partitionNameAfter:[{}]",
+
logInfo("catalogName:[{}],dbName:[{}],tableName:[{}],partitionNameBefore:[{}],partitionNameAfter:[{}]",
catalogName, dbName, tblName, partitionNameBefore,
partitionNameAfter);
if (isRename) {
Env.getCurrentEnv().getCatalogMgr()
@@ -125,7 +125,7 @@ public class AlterPartitionEvent extends
MetastorePartitionEvent {
}
} catch (DdlException e) {
throw new MetastoreNotificationException(
- debugString("Failed to process event"), e);
+ getMsgWithEventInfo("Failed to process event"), e);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java
index 2283646b64a..43999fac8ce 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java
@@ -59,7 +59,7 @@ public class AlterTableEvent extends MetastoreTableEvent {
super(event, catalogName);
Preconditions.checkArgument(MetastoreEventType.ALTER_TABLE.equals(getEventType()));
Preconditions
- .checkNotNull(event.getMessage(), debugString("Event message
is null"));
+ .checkNotNull(event.getMessage(), getMsgWithEventInfo("Event
message is null"));
try {
JSONAlterTableMessage alterTableMessage =
(JSONAlterTableMessage)
MetastoreEventsProcessor.getMessageDeserializer(event.getMessageFormat())
@@ -70,7 +70,7 @@ public class AlterTableEvent extends MetastoreTableEvent {
tblNameAfter = tableAfter.getTableName();
} catch (Exception e) {
throw new MetastoreNotificationException(
- debugString("Unable to parse the alter table message"), e);
+ getMsgWithEventInfo("Unable to parse the alter table
message"), e);
}
// this is a rename event if either dbName or tblName of before and
after object changed
isRename =
!tableBefore.getDbName().equalsIgnoreCase(tableAfter.getDbName())
@@ -111,7 +111,7 @@ public class AlterTableEvent extends MetastoreTableEvent {
boolean hasExist = Env.getCurrentEnv().getCatalogMgr()
.externalTableExistInLocal(tableAfter.getDbName(),
tableAfter.getTableName(), catalogName);
if (hasExist) {
- infoLog("AlterExternalTable canceled,because tableAfter has exist,
"
+ logInfo("AlterExternalTable canceled,because tableAfter has exist,
"
+ "catalogName:[{}],dbName:[{}],tableName:[{}]",
catalogName, dbName, tableAfter.getTableName());
return;
@@ -143,7 +143,7 @@ public class AlterTableEvent extends MetastoreTableEvent {
@Override
protected void process() throws MetastoreNotificationException {
try {
-
infoLog("catalogName:[{}],dbName:[{}],tableBefore:[{}],tableAfter:[{}]",
catalogName, dbName,
+
logInfo("catalogName:[{}],dbName:[{}],tableBefore:[{}],tableAfter:[{}]",
catalogName, dbName,
tableBefore.getTableName(), tableAfter.getTableName());
if (isRename) {
processRename();
@@ -161,7 +161,7 @@ public class AlterTableEvent extends MetastoreTableEvent {
eventTime);
} catch (Exception e) {
throw new MetastoreNotificationException(
- debugString("Failed to process event"), e);
+ getMsgWithEventInfo("Failed to process event"), e);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java
index 2dd4c5671bc..b11a9eaa072 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java
@@ -54,11 +54,11 @@ public class CreateDatabaseEvent extends MetastoreEvent {
@Override
protected void process() throws MetastoreNotificationException {
try {
- infoLog("catalogName:[{}],dbName:[{}]", catalogName, dbName);
+ logInfo("catalogName:[{}],dbName:[{}]", catalogName, dbName);
Env.getCurrentEnv().getCatalogMgr().registerExternalDatabaseFromEvent(dbName,
catalogName, true);
} catch (DdlException e) {
throw new MetastoreNotificationException(
- debugString("Failed to process event"), e);
+ getMsgWithEventInfo("Failed to process event"), e);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateTableEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateTableEvent.java
index e6c3e2e7eae..8a22a479ad7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateTableEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateTableEvent.java
@@ -48,7 +48,7 @@ public class CreateTableEvent extends MetastoreTableEvent {
super(event, catalogName);
Preconditions.checkArgument(MetastoreEventType.CREATE_TABLE.equals(getEventType()));
Preconditions
- .checkNotNull(event.getMessage(), debugString("Event message
is null"));
+ .checkNotNull(event.getMessage(), getMsgWithEventInfo("Event
message is null"));
try {
CreateTableMessage createTableMessage =
MetastoreEventsProcessor.getMessageDeserializer(event.getMessageFormat())
@@ -57,7 +57,7 @@ public class CreateTableEvent extends MetastoreTableEvent {
hmsTbl.setTableName(hmsTbl.getTableName().toLowerCase(Locale.ROOT));
} catch (Exception e) {
throw new MetastoreNotificationException(
- debugString("Unable to deserialize the event message"), e);
+ getMsgWithEventInfo("Unable to deserialize the event
message"), e);
}
}
@@ -78,12 +78,12 @@ public class CreateTableEvent extends MetastoreTableEvent {
@Override
protected void process() throws MetastoreNotificationException {
try {
- infoLog("catalogName:[{}],dbName:[{}],tableName:[{}]",
catalogName, dbName, tblName);
+ logInfo("catalogName:[{}],dbName:[{}],tableName:[{}]",
catalogName, dbName, tblName);
Env.getCurrentEnv().getCatalogMgr()
.registerExternalTableFromEvent(dbName,
hmsTbl.getTableName(), catalogName, eventTime, true);
} catch (DdlException e) {
throw new MetastoreNotificationException(
- debugString("Failed to process event"), e);
+ getMsgWithEventInfo("Failed to process event"), e);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java
index 107ce591a42..0649106661e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java
@@ -53,12 +53,12 @@ public class DropDatabaseEvent extends MetastoreEvent {
@Override
protected void process() throws MetastoreNotificationException {
try {
- infoLog("catalogName:[{}],dbName:[{}]", catalogName, dbName);
+ logInfo("catalogName:[{}],dbName:[{}]", catalogName, dbName);
Env.getCurrentEnv().getCatalogMgr()
.unregisterExternalDatabase(dbName, catalogName, true);
} catch (DdlException e) {
throw new MetastoreNotificationException(
- debugString("Failed to process event"), e);
+ getMsgWithEventInfo("Failed to process event"), e);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java
index dd443010289..737ad8f28b9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java
@@ -57,7 +57,7 @@ public class DropPartitionEvent extends
MetastorePartitionEvent {
super(event, catalogName);
Preconditions.checkArgument(getEventType().equals(MetastoreEventType.DROP_PARTITION));
Preconditions
- .checkNotNull(event.getMessage(), debugString("Event message
is null"));
+ .checkNotNull(event.getMessage(), getMsgWithEventInfo("Event
message is null"));
try {
DropPartitionMessage dropPartitionMessage =
MetastoreEventsProcessor.getMessageDeserializer(event.getMessageFormat())
@@ -97,11 +97,11 @@ public class DropPartitionEvent extends
MetastorePartitionEvent {
@Override
protected void process() throws MetastoreNotificationException {
try {
-
infoLog("catalogName:[{}],dbName:[{}],tableName:[{}],partitionNames:[{}]",
catalogName, dbName, tblName,
+
logInfo("catalogName:[{}],dbName:[{}],tableName:[{}],partitionNames:[{}]",
catalogName, dbName, tblName,
partitionNames.toString());
// bail out early if there are not partitions to process
if (partitionNames.isEmpty()) {
- infoLog("Partition list is empty. Ignoring this event.");
+ logInfo("Partition list is empty. Ignoring this event.");
return;
}
Env.getCurrentEnv().getCatalogMgr()
@@ -109,7 +109,7 @@ public class DropPartitionEvent extends
MetastorePartitionEvent {
partitionNames, eventTime, true);
} catch (DdlException e) {
throw new MetastoreNotificationException(
- debugString("Failed to process event"), e);
+ getMsgWithEventInfo("Failed to process event"), e);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropTableEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropTableEvent.java
index 6dcb16dedad..dd67d660527 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropTableEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropTableEvent.java
@@ -48,7 +48,7 @@ public class DropTableEvent extends MetastoreTableEvent {
super(event, catalogName);
Preconditions.checkArgument(MetastoreEventType.DROP_TABLE.equals(getEventType()));
Preconditions
- .checkNotNull(event.getMessage(), debugString("Event message
is null"));
+ .checkNotNull(event.getMessage(), getMsgWithEventInfo("Event
message is null"));
try {
JSONDropTableMessage dropTableMessage =
(JSONDropTableMessage)
MetastoreEventsProcessor.getMessageDeserializer(event.getMessageFormat())
@@ -77,11 +77,11 @@ public class DropTableEvent extends MetastoreTableEvent {
@Override
protected void process() throws MetastoreNotificationException {
try {
- infoLog("catalogName:[{}],dbName:[{}],tableName:[{}]",
catalogName, dbName, tableName);
+ logInfo("catalogName:[{}],dbName:[{}],tableName:[{}]",
catalogName, dbName, tableName);
Env.getCurrentEnv().getCatalogMgr().unregisterExternalTable(dbName, tableName,
catalogName, true);
} catch (DdlException e) {
throw new MetastoreNotificationException(
- debugString("Failed to process event"), e);
+ getMsgWithEventInfo("Failed to process event"), e);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/IgnoredEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/IgnoredEvent.java
index e7e6643e647..bebfc8c2384 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/IgnoredEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/IgnoredEvent.java
@@ -38,6 +38,6 @@ public class IgnoredEvent extends MetastoreEvent {
@Override
public void process() {
- infoLog("Ignoring unknown event type " +
metastoreNotificationEvent.getEventType());
+ logInfo("Ignoring unknown event type " +
metastoreNotificationEvent.getEventType());
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java
index 7b76d4913d5..4b4abd0264d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java
@@ -42,7 +42,7 @@ public class InsertEvent extends MetastoreTableEvent {
super(event, catalogName);
Preconditions.checkArgument(getEventType().equals(MetastoreEventType.INSERT));
Preconditions
- .checkNotNull(event.getMessage(), debugString("Event message
is null"));
+ .checkNotNull(event.getMessage(), getMsgWithEventInfo("Event
message is null"));
}
protected static List<MetastoreEvent> getEvents(NotificationEvent event,
String catalogName) {
@@ -62,7 +62,7 @@ public class InsertEvent extends MetastoreTableEvent {
@Override
protected void process() throws MetastoreNotificationException {
try {
- infoLog("catalogName:[{}],dbName:[{}],tableName:[{}]",
catalogName, dbName, tblName);
+ logInfo("catalogName:[{}],dbName:[{}],tableName:[{}]",
catalogName, dbName, tblName);
/**
* Only when we use hive client to execute a `INSERT INTO TBL
SELECT * ...` or `INSERT INTO TBL ...` sql
* to a non-partitioned table then the hms will generate an
insert event, and there is not
@@ -75,7 +75,7 @@ public class InsertEvent extends MetastoreTableEvent {
eventTime);
} catch (DdlException e) {
throw new MetastoreNotificationException(
- debugString("Failed to process event"), e);
+ getMsgWithEventInfo("Failed to process event"), e);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEvent.java
index 695dd57b215..b0ec23b0661 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEvent.java
@@ -18,7 +18,6 @@
package org.apache.doris.datasource.hive.event;
import org.apache.doris.datasource.MetaIdMappingsLog;
-import org.apache.doris.datasource.hive.HMSCachedClient;
import com.google.common.collect.ImmutableList;
import org.apache.commons.lang3.StringUtils;
@@ -31,43 +30,29 @@ import java.util.Locale;
import java.util.Map;
/**
- * Abstract base class for all MetastoreEvents. A MetastoreEvent is an object
used to
- * process a NotificationEvent received from metastore.
+ * The wrapper parent class of the NotificationEvent class
*/
public abstract class MetastoreEvent {
private static final Logger LOG =
LogManager.getLogger(MetastoreEvent.class);
- // String.format compatible string to prepend event id and type
- private static final String STR_FORMAT_EVENT_ID_TYPE = "EventId: %d
EventType: %s ";
- // logger format compatible string to prepend to a log formatted message
- private static final String LOG_FORMAT_EVENT_ID_TYPE = "EventId: {}
EventType: {} ";
-
- // the notification received from metastore which is processed by this
- protected final NotificationEvent event;
+ protected final String catalogName;
- // dbName from the event
protected final String dbName;
- // tblName from the event
protected final String tblName;
- // eventId of the event. Used instead of calling getter on event everytime
protected final long eventId;
- // eventTime of the event. Used instead of calling getter on event
everytime
protected final long eventTime;
- // eventType from the NotificationEvent
protected final MetastoreEventType eventType;
- // Actual notificationEvent object received from Metastore
+ protected final NotificationEvent event;
protected final NotificationEvent metastoreNotificationEvent;
- protected final String catalogName;
-
// for test
protected MetastoreEvent(long eventId, String catalogName, String dbName,
- String tblName, MetastoreEventType eventType) {
+ String tblName, MetastoreEventType eventType) {
this.eventId = eventId;
this.eventTime = -1L;
this.catalogName = catalogName;
@@ -103,127 +88,46 @@ public abstract class MetastoreEvent {
this.catalogName = catalogName;
}
- public long getEventId() {
- return eventId;
- }
-
- public MetastoreEventType getEventType() {
- return eventType;
- }
-
- public String getDbName() {
- return dbName;
- }
-
- public String getTblName() {
- return tblName;
- }
-
/**
- * Checks if the given event can be batched into this event. Default
behavior is
- * to return false which can be overridden by a sub-class.
+ * Can batch processing be performed to improve processing performance
*
- * @param event The event under consideration to be batched into this
event.
- * @return false if event cannot be batched into this event; otherwise
true.
+ * @param event
+ * @return
*/
protected boolean canBeBatched(MetastoreEvent event) {
return false;
}
- /**
- * Adds the given event into the batch of events represented by this
event. Default
- * implementation is to return null. Sub-classes must override this method
to
- * implement batching.
- *
- * @param event The event which needs to be added to the batch.
- * @return The batch event which represents all the events batched into
this event
- * until now including the given event.
- */
- protected MetastoreEvent addToBatchEvents(MetastoreEvent event) {
- return null;
- }
-
- /**
- * Returns the number of events represented by this event. For most events
this is 1.
- * In case of batch events this could be more than 1.
- */
- protected int getNumberOfEvents() {
- return 1;
- }
-
- /**
- * Certain events like ALTER_TABLE or ALTER_PARTITION implement logic to
ignore
- * some events because they do not affect query results.
- *
- * @return true if this event can be skipped.
- */
- protected boolean canBeSkipped() {
- return false;
- }
-
- /**
- * Process the information available in the NotificationEvent.
- * Better not to call (direct/indirect) apis of {@link HMSCachedClient}
- * during handling hms events (Reference to
https://github.com/apache/doris/pull/19120).
- * Try to add some fallback strategies if it is highly necessary.
- */
protected abstract void process() throws MetastoreNotificationException;
- /**
- * Helper method to get debug string with helpful event information
prepended to the
- * message. This can be used to generate helpful exception messages
- *
- * @param msgFormatString String value to be used in String.format() for
the given message
- * @param args args to the <code>String.format()</code> for the given
msgFormatString
- */
- protected String debugString(String msgFormatString, Object... args) {
- String formatString = STR_FORMAT_EVENT_ID_TYPE + msgFormatString;
- Object[] formatArgs = getLogFormatArgs(args);
- return String.format(formatString, formatArgs);
- }
-
- /**
- * Helper method to generate the format args after prepending the event id
and type
- */
- private Object[] getLogFormatArgs(Object[] args) {
- Object[] formatArgs = new Object[args.length + 2];
- formatArgs[0] = eventId;
- formatArgs[1] = eventType;
- int i = 2;
- for (Object arg : args) {
- formatArgs[i] = arg;
- i++;
- }
- return formatArgs;
+ protected String getMsgWithEventInfo(String formatSuffix, Object... args) {
+ String format = "EventId: %d EventType: %s " + formatSuffix;
+ Object[] argsWithEventInfo = getArgsWithEventInfo(args);
+ return String.format(format, argsWithEventInfo);
}
- /**
- * Logs at info level the given log formatted string and its args. The log
formatted
- * string should have {} pair at the appropriate location in the string
for each arg
- * value provided. This method prepends the event id and event type before
logging the
- * message. No-op if the log level is not at INFO
- */
- protected void infoLog(String logFormattedStr, Object... args) {
+ protected void logInfo(String formatSuffix, Object... args) {
if (!LOG.isInfoEnabled()) {
return;
}
- String formatString = LOG_FORMAT_EVENT_ID_TYPE + logFormattedStr;
- Object[] formatArgs = getLogFormatArgs(args);
- LOG.info(formatString, formatArgs);
+ String format = "EventId: {} EventType: {} " + formatSuffix;
+ Object[] argsWithEventInfo = getArgsWithEventInfo(args);
+ LOG.info(format, argsWithEventInfo);
}
/**
- * Similar to infoLog excepts logs at debug level
+ * Add event information to the parameters
*/
- protected void debugLog(String logFormattedStr, Object... args) {
- if (!LOG.isDebugEnabled()) {
- return;
- }
- String formatString = LOG_FORMAT_EVENT_ID_TYPE + logFormattedStr;
- Object[] formatArgs = getLogFormatArgs(args);
- if (LOG.isDebugEnabled()) {
- LOG.debug(formatString, formatArgs);
+ private Object[] getArgsWithEventInfo(Object[] args) {
+ Object[] res = new Object[args.length + 2];
+ res[0] = eventId;
+ res[1] = eventType;
+ int i = 2;
+ for (Object arg : args) {
+ res[i] = arg;
+ i++;
}
+ return res;
}
protected String getPartitionName(Map<String, String> part, List<String>
partitionColNames) {
@@ -253,8 +157,27 @@ public abstract class MetastoreEvent {
return ImmutableList.of();
}
+ public String getDbName() {
+ return dbName;
+ }
+
+ public String getTblName() {
+ return tblName;
+ }
+
+ public long getEventId() {
+ return eventId;
+ }
+
+ public MetastoreEventType getEventType() {
+ return eventType;
+ }
+
@Override
public String toString() {
- return String.format(STR_FORMAT_EVENT_ID_TYPE, eventId, eventType);
+ return "MetastoreEvent{"
+ + "eventId=" + eventId
+ + ", eventType=" + eventType
+ + '}';
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java
index cbd0bfb5fa6..73054773402 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java
@@ -165,7 +165,7 @@ public class MetastoreEventsProcessor extends MasterDaemon {
} catch (HMSClientException hmsClientException) {
if (hmsClientException.getCause() != null
&& hmsClientException.getCause() instanceof
NoSuchObjectException) {
- LOG.warn(event.debugString("Failed to process event and
skip"), hmsClientException);
+ LOG.warn(event.getMsgWithEventInfo("Failed to process
event and skip"), hmsClientException);
} else {
updateLastSyncedEventId(hmsExternalCatalog,
event.getEventId() - 1);
throw hmsClientException;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
