This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new 34410276b2 Aggregate slow statement records by service dimension
(#13514)
34410276b2 is described below
commit 34410276b2b88b970e5eb9e771f1a10e9d1577f1
Author: peachisai <[email protected]>
AuthorDate: Thu Sep 25 10:21:50 2025 +0800
Aggregate slow statement records by service dimension (#13514)
---
docs/en/changes/changes.md | 1 +
.../vservice/VirtualDatabaseProcessor.java | 37 +++++++--
.../vservice/VirtualDatabaseProcessorTest.java | 11 ++-
.../ServiceDatabaseSlowStatementDispatcher.java | 39 +++++++++
.../database/TopNServiceDatabaseStatement.java | 96 ++++++++++++++++++++++
.../oap/server/core/source/DefaultScopeDefine.java | 1 +
.../core/source/ServiceDatabaseSlowStatement.java | 67 +++++++++++++++
.../general/general-service.json | 26 ++++++
8 files changed, 268 insertions(+), 10 deletions(-)
diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
index 470752b29e..ec986593d4 100644
--- a/docs/en/changes/changes.md
+++ b/docs/en/changes/changes.md
@@ -101,6 +101,7 @@
* Bump up netty to 4.2.5.Final.
* BanyanDB: fix log query missing order by condition, and fix missing service
id condition when query by instance id or endpoint id.
* Fix potential NPE in the `AlarmStatusQueryHandler`.
+* Aggregate TopN Slow SQL by service dimension.
#### UI
diff --git
a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualDatabaseProcessor.java
b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualDatabaseProcessor.java
index 9e8c7df2cd..8ee113cb79 100644
---
a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualDatabaseProcessor.java
+++
b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualDatabaseProcessor.java
@@ -33,6 +33,7 @@ import
org.apache.skywalking.oap.server.core.analysis.TimeBucket;
import org.apache.skywalking.oap.server.core.config.NamingControl;
import org.apache.skywalking.oap.server.core.source.DatabaseAccess;
import org.apache.skywalking.oap.server.core.source.DatabaseSlowStatement;
+import
org.apache.skywalking.oap.server.core.source.ServiceDatabaseSlowStatement;
import org.apache.skywalking.oap.server.core.source.ServiceMeta;
import org.apache.skywalking.oap.server.core.source.Source;
import org.apache.skywalking.oap.server.library.util.StringUtil;
@@ -65,18 +66,36 @@ public class VirtualDatabaseProcessor implements
VirtualServiceProcessor {
recordList.add(toDatabaseAccess(span, serviceName, timeBucket,
latency));
readStatementIfSlow(span.getTagsList(), latency).ifPresent(statement
-> {
- DatabaseSlowStatement dbSlowStat = new DatabaseSlowStatement();
- dbSlowStat.setId(segmentObject.getTraceSegmentId() + "-" +
span.getSpanId());
- dbSlowStat.setTraceId(segmentObject.getTraceId());
-
dbSlowStat.setDatabaseServiceId(IDManager.ServiceID.buildId(serviceName,
false));
- dbSlowStat.setStatement(statement);
- dbSlowStat.setLatency(latency);
-
dbSlowStat.setTimeBucket(TimeBucket.getRecordTimeBucket(span.getStartTime()));
- dbSlowStat.setTimestamp(span.getStartTime());
- recordList.add(dbSlowStat);
+ recordList.add(buildDatabaseSlowStatement(span, segmentObject,
statement, serviceName, latency));
+ recordList.add(buildServiceDatabaseSlowStatement(span,
segmentObject, statement, latency));
});
}
+ private DatabaseSlowStatement buildDatabaseSlowStatement(SpanObject span,
SegmentObject segmentObject, String statement, String serviceName, int latency)
{
+ DatabaseSlowStatement dbSlowStat = new DatabaseSlowStatement();
+ dbSlowStat.setId(segmentObject.getTraceSegmentId() + "-" +
span.getSpanId());
+ dbSlowStat.setTraceId(segmentObject.getTraceId());
+
dbSlowStat.setDatabaseServiceId(IDManager.ServiceID.buildId(serviceName,
false));
+ dbSlowStat.setStatement(statement);
+ dbSlowStat.setLatency(latency);
+
dbSlowStat.setTimeBucket(TimeBucket.getRecordTimeBucket(span.getStartTime()));
+ dbSlowStat.setTimestamp(span.getStartTime());
+ return dbSlowStat;
+ }
+
+ private ServiceDatabaseSlowStatement
buildServiceDatabaseSlowStatement(SpanObject span, SegmentObject segmentObject,
String statement, int latency) {
+ ServiceDatabaseSlowStatement serviceDbSlowStat = new
ServiceDatabaseSlowStatement();
+ serviceDbSlowStat.setId(segmentObject.getTraceSegmentId() + "-" +
span.getSpanId());
+ serviceDbSlowStat.setTraceId(segmentObject.getTraceId());
+
serviceDbSlowStat.setServiceId(IDManager.ServiceID.buildId(segmentObject.getService(),
true));
+ serviceDbSlowStat.setStatement(statement);
+ serviceDbSlowStat.setLatency(latency);
+
serviceDbSlowStat.setTimeBucket(TimeBucket.getRecordTimeBucket(span.getStartTime()));
+ serviceDbSlowStat.setTimestamp(span.getStartTime());
+
+ return serviceDbSlowStat;
+ }
+
private Optional<String> readStatementIfSlow(List<KeyStringValuePair>
tags, int latency) {
String statement = null;
boolean isSlowDBAccess = false;
diff --git
a/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualDatabaseProcessorTest.java
b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualDatabaseProcessorTest.java
index a64c2c20e8..3b4ea7db40 100644
---
a/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualDatabaseProcessorTest.java
+++
b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualDatabaseProcessorTest.java
@@ -32,6 +32,7 @@ import
org.apache.skywalking.oap.server.core.config.NamingControl;
import org.apache.skywalking.oap.server.core.config.group.EndpointNameGrouping;
import org.apache.skywalking.oap.server.core.source.DatabaseAccess;
import org.apache.skywalking.oap.server.core.source.DatabaseSlowStatement;
+import
org.apache.skywalking.oap.server.core.source.ServiceDatabaseSlowStatement;
import org.apache.skywalking.oap.server.core.source.ServiceMeta;
import org.apache.skywalking.oap.server.core.source.Source;
import org.joda.time.DateTime;
@@ -70,12 +71,13 @@ public class VirtualDatabaseProcessorTest {
.build();
SegmentObject segmentObject = SegmentObject.newBuilder()
.setTraceId("trace-id-1")
+ .setService("test-service")
.build();
VirtualDatabaseProcessor processor = buildVirtualServiceProcessor();
processor.prepareVSIfNecessary(spanObject, segmentObject);
ArrayList<Source> sources = new ArrayList<>();
processor.emitTo(sources::add);
- Assertions.assertEquals(sources.size(), 3);
+ Assertions.assertEquals(sources.size(), 4);
ServiceMeta serviceMeta = (ServiceMeta) sources.get(0);
Assertions.assertEquals("127.0.0.1:3306", serviceMeta.getName());
@@ -92,6 +94,13 @@ public class VirtualDatabaseProcessorTest {
Assertions.assertEquals(1000, slowStatement.getLatency());
Assertions.assertEquals(20220912141312L,
slowStatement.getTimeBucket());
Assertions.assertEquals("trace-id-1", slowStatement.getTraceId());
+
+ ServiceDatabaseSlowStatement serviceDatabaseSlowStatement =
(ServiceDatabaseSlowStatement) sources.get(3);
+ Assertions.assertEquals("dGVzdC1zZXJ2aWNl.1",
serviceDatabaseSlowStatement.getServiceId());
+ Assertions.assertEquals(1000, slowStatement.getLatency());
+ Assertions.assertEquals(20220912141312L,
slowStatement.getTimeBucket());
+ Assertions.assertEquals("trace-id-1", slowStatement.getTraceId());
+
}
@Test
diff --git
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/database/ServiceDatabaseSlowStatementDispatcher.java
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/database/ServiceDatabaseSlowStatementDispatcher.java
new file mode 100644
index 0000000000..52dc4fcb66
--- /dev/null
+++
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/database/ServiceDatabaseSlowStatementDispatcher.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.analysis.manual.database;
+
+import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher;
+import
org.apache.skywalking.oap.server.core.analysis.worker.TopNStreamProcessor;
+import
org.apache.skywalking.oap.server.core.source.ServiceDatabaseSlowStatement;
+
+public class ServiceDatabaseSlowStatementDispatcher implements
SourceDispatcher<ServiceDatabaseSlowStatement> {
+ @Override
+ public void dispatch(ServiceDatabaseSlowStatement source) {
+ TopNServiceDatabaseStatement statement = new
TopNServiceDatabaseStatement();
+ statement.setId(source.getId());
+ statement.setEntityId(source.getServiceId());
+ statement.setLatency(source.getLatency());
+ statement.setStatement(source.getStatement());
+ statement.setTimeBucket(source.getTimeBucket());
+ statement.setTraceId(source.getTraceId());
+ statement.setTimestamp(source.getTimestamp());
+
+ TopNStreamProcessor.getInstance().in(statement);
+ }
+}
diff --git
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/database/TopNServiceDatabaseStatement.java
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/database/TopNServiceDatabaseStatement.java
new file mode 100644
index 0000000000..d3e14df3b6
--- /dev/null
+++
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/database/TopNServiceDatabaseStatement.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.analysis.manual.database;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.skywalking.oap.server.core.analysis.Stream;
+import org.apache.skywalking.oap.server.core.analysis.topn.TopN;
+import
org.apache.skywalking.oap.server.core.analysis.worker.TopNStreamProcessor;
+import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
+import org.apache.skywalking.oap.server.core.storage.StorageID;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB;
+import org.apache.skywalking.oap.server.core.storage.annotation.Column;
+import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
+import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
+import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder;
+
+import java.util.Objects;
+
+/**
+ * Service Database TopN statement, including Database SQL statement, mongoDB
and Redis commands.
+ */
+@Stream(name = TopNServiceDatabaseStatement.INDEX_NAME, scopeId =
DefaultScopeDefine.SERVICE_DATABASE_SLOW_STATEMENT, builder =
TopNServiceDatabaseStatement.Builder.class, processor =
TopNStreamProcessor.class)
[email protected](TopN.TIMESTAMP)
[email protected](streamGroup = BanyanDB.StreamGroup.RECORDS)
+public class TopNServiceDatabaseStatement extends TopN {
+
+ public static final String INDEX_NAME = "top_n_service_database_statement";
+
+ @Setter
+ private String id;
+ @Getter
+ @Setter
+ @Column(name = STATEMENT, length = 2000, storageOnly = true)
+ private String statement;
+
+ @Override
+ public StorageID id() {
+ return new StorageID().append(id);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+ TopNServiceDatabaseStatement statement =
(TopNServiceDatabaseStatement) o;
+ return Objects.equals(getEntityId(), statement.getEntityId());
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(getEntityId());
+ }
+
+ public static class Builder implements
StorageBuilder<TopNServiceDatabaseStatement> {
+ @Override
+ public TopNServiceDatabaseStatement storage2Entity(final
Convert2Entity converter) {
+ TopNServiceDatabaseStatement statement = new
TopNServiceDatabaseStatement();
+ statement.setStatement((String) converter.get(STATEMENT));
+ statement.setTraceId((String) converter.get(TRACE_ID));
+ statement.setLatency(((Number)
converter.get(LATENCY)).longValue());
+ statement.setEntityId((String) converter.get(ENTITY_ID));
+ statement.setTimeBucket(((Number)
converter.get(TIME_BUCKET)).longValue());
+ statement.setTimestamp(((Number)
converter.get(TIMESTAMP)).longValue());
+ return statement;
+ }
+
+ @Override
+ public void entity2Storage(final TopNServiceDatabaseStatement
storageData, final Convert2Storage converter) {
+ converter.accept(STATEMENT, storageData.getStatement());
+ converter.accept(TRACE_ID, storageData.getTraceId());
+ converter.accept(LATENCY, storageData.getLatency());
+ converter.accept(ENTITY_ID, storageData.getEntityId());
+ converter.accept(TIME_BUCKET, storageData.getTimeBucket());
+ converter.accept(TIMESTAMP, storageData.getTimestamp());
+ }
+ }
+}
diff --git
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java
index 1d51cc193e..71fe59b4e0 100644
---
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java
+++
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java
@@ -151,6 +151,7 @@ public class DefaultScopeDefine {
public static final int BROWSER_APP_RESOURCE_PERF = 88;
public static final int BROWSER_APP_WEB_INTERACTION_PAGE_PERF = 89;
public static final int SW_SPAN_ATTACHED_EVENT = 90;
+ public static final int SERVICE_DATABASE_SLOW_STATEMENT = 91;
/**
* Catalog of scope, the metrics processor could use this to group all
generated metrics by oal rt.
diff --git
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceDatabaseSlowStatement.java
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceDatabaseSlowStatement.java
new file mode 100644
index 0000000000..a15258bb68
--- /dev/null
+++
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceDatabaseSlowStatement.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.source;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.skywalking.oap.server.core.Const;
+
+import static
org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SERVICE_CATALOG_NAME;
+import static
org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SERVICE_DATABASE_SLOW_STATEMENT;
+
+@ScopeDeclaration(id = SERVICE_DATABASE_SLOW_STATEMENT, name =
"ServiceDatabaseSlowStatement", catalog = SERVICE_CATALOG_NAME)
+public class ServiceDatabaseSlowStatement extends Source {
+
+ @Getter
+ @Setter
+ private String id;
+
+ @Getter
+ @Setter
+ @ScopeDefaultColumn.DefinedByField(columnName = "service_id")
+ @ScopeDefaultColumn.BanyanDB(shardingKeyIdx = 0)
+ private String serviceId;
+
+ @Getter
+ @Setter
+ private String statement;
+
+ @Getter
+ @Setter
+ private long latency;
+
+ @Getter
+ @Setter
+ private String traceId;
+
+ @Getter
+ @Setter
+ private long timestamp;
+
+ @Override
+ public int scope() {
+ return DefaultScopeDefine.SERVICE_DATABASE_SLOW_STATEMENT;
+ }
+
+ @Override
+ public String getEntityId() {
+ return Const.EMPTY_STRING;
+ }
+
+}
diff --git
a/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-service.json
b/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-service.json
index 45f46477d4..86ea846249 100644
---
a/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-service.json
+++
b/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-service.json
@@ -860,6 +860,32 @@
"type": "Log"
}
]
+ },
+ {
+ "name": "Slow Statements",
+ "children": [
+ {
+ "x": 0,
+ "y": 0,
+ "w": 24,
+ "h": 48,
+ "i": "0",
+ "type": "Widget",
+ "widget": {
+ "title": "Slow Statements (ms)"
+ },
+ "graph": {
+ "type": "TopList",
+ "color": "purple"
+ },
+ "expressions": [
+ "top_n(top_n_service_database_statement,20,des)"
+ ],
+ "relatedTrace": {
+ "refIdType": "traceId"
+ }
+ }
+ ]
}
]
},
