This is an automated email from the ASF dual-hosted git repository.
yihaochen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new 56cfa52d87 Remove `matchedCounter` from
`HttpUriRecognitionService#feedRawData` (#10966)
56cfa52d87 is described below
commit 56cfa52d872079c8689fa2cf31f57d7200bbc604
Author: 吴晟 Wu Sheng <wu.sh...@foxmail.com>
AuthorDate: Wed Jun 21 01:17:05 2023 +0800
Remove `matchedCounter` from `HttpUriRecognitionService#feedRawData`
(#10966)
* [Breaking change] Remove `matchedCounter` from
`HttpUriRecognitionService#feedRawData`.
* * Remove patterns from `HttpUriRecognitionService#feedRawData` and add
max 10 candidates of raw URIs for each pattern.
---------
Co-authored-by: kezhenxu94 <kezhenx...@apache.org>
---
.github/workflows/skywalking.yaml | 4 +-
docs/en/changes/changes.md | 2 +
.../services/HttpUriRecognitionService.java | 2 +-
.../pipeline/services/api/HttpUriRecognition.java | 3 +-
.../src/main/proto/ai_http_uri_recognition.proto | 1 -
.../core/config/group/EndpointNameGrouping.java | 54 ++++++++++++++--------
test/e2e-v2/script/env | 2 +-
7 files changed, 42 insertions(+), 26 deletions(-)
diff --git a/.github/workflows/skywalking.yaml
b/.github/workflows/skywalking.yaml
index e4f80444d6..52226fb86e 100644
--- a/.github/workflows/skywalking.yaml
+++ b/.github/workflows/skywalking.yaml
@@ -750,8 +750,8 @@ jobs:
matrix:
analyzer: [k8s-mesh, mx-mesh]
versions:
- - istio: 1.7.1
- kubernetes: 18
+ #- istio: 1.7.1
+ # kubernetes: 18
- istio: 1.8.2
kubernetes: 19
- istio: 1.9.1
diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
index 5d900a14fc..dc22c946a5 100644
--- a/docs/en/changes/changes.md
+++ b/docs/en/changes/changes.md
@@ -10,6 +10,8 @@
* Add Istio ServiceEntry registry to resolve unknown IPs in ALS.
* Improve Kubernetes coordinator to only select ready OAP Pods to build
cluster.
* Wrap `deleteProperty` API to the BanyanDBStorageClient.
+* [Breaking change] Remove `matchedCounter` from
`HttpUriRecognitionService#feedRawData`.
+* Remove patterns from `HttpUriRecognitionService#feedRawData` and add max 10
candidates of raw URIs for each pattern.
#### UI
diff --git
a/oap-server/ai-pipeline/src/main/java/org/apache/skywalking/oap/server/ai/pipeline/services/HttpUriRecognitionService.java
b/oap-server/ai-pipeline/src/main/java/org/apache/skywalking/oap/server/ai/pipeline/services/HttpUriRecognitionService.java
index 66d5e5bfe8..51a59821d7 100644
---
a/oap-server/ai-pipeline/src/main/java/org/apache/skywalking/oap/server/ai/pipeline/services/HttpUriRecognitionService.java
+++
b/oap-server/ai-pipeline/src/main/java/org/apache/skywalking/oap/server/ai/pipeline/services/HttpUriRecognitionService.java
@@ -93,7 +93,7 @@ public class HttpUriRecognitionService implements
HttpUriRecognition {
builder.setService(service);
unrecognizedURIs.forEach(httpUri -> {
builder.getUnrecognizedUrisBuilderList().add(
-
HttpRawUri.newBuilder().setName(httpUri.getName()).setMatchedCounter(httpUri.getMatchedCounter())
+ HttpRawUri.newBuilder().setName(httpUri.getName())
);
});
stub.withDeadlineAfter(30, TimeUnit.SECONDS)
diff --git
a/oap-server/ai-pipeline/src/main/java/org/apache/skywalking/oap/server/ai/pipeline/services/api/HttpUriRecognition.java
b/oap-server/ai-pipeline/src/main/java/org/apache/skywalking/oap/server/ai/pipeline/services/api/HttpUriRecognition.java
index 51d299ccb9..40daefc1b5 100644
---
a/oap-server/ai-pipeline/src/main/java/org/apache/skywalking/oap/server/ai/pipeline/services/api/HttpUriRecognition.java
+++
b/oap-server/ai-pipeline/src/main/java/org/apache/skywalking/oap/server/ai/pipeline/services/api/HttpUriRecognition.java
@@ -44,7 +44,7 @@ public interface HttpUriRecognition extends Service {
* Feed all data for the pattern recognition of HTTP URIs
*
* @param service the name of the service
- * @param unrecognizedURIs the list of unrecognized URIs
+ * @param unrecognizedURIs the list of unrecognized URIs and candidates of
recognized URIs.
*/
void feedRawData(String service, List<HTTPUri> unrecognizedURIs);
@@ -52,6 +52,5 @@ public interface HttpUriRecognition extends Service {
@Getter
class HTTPUri {
private final String name;
- private final long matchedCounter;
}
}
diff --git
a/oap-server/ai-pipeline/src/main/proto/ai_http_uri_recognition.proto
b/oap-server/ai-pipeline/src/main/proto/ai_http_uri_recognition.proto
index f5cb57eb30..b03e7e7323 100644
--- a/oap-server/ai-pipeline/src/main/proto/ai_http_uri_recognition.proto
+++ b/oap-server/ai-pipeline/src/main/proto/ai_http_uri_recognition.proto
@@ -44,7 +44,6 @@ message HttpUriRecognitionRequest {
message HttpRawUri {
string name = 1;
- int64 matchedCounter = 2;
}
message HttpUriRecognitionResponse {
diff --git
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java
index abbb51d8e2..a2c1bc8e0d 100644
---
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java
+++
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java
@@ -20,12 +20,13 @@ package org.apache.skywalking.oap.server.core.config.group;
import io.vavr.Tuple2;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import
org.apache.skywalking.oap.server.ai.pipeline.services.api.HttpUriPattern;
@@ -56,12 +57,11 @@ public class EndpointNameGrouping {
private volatile QuickUriGroupingRule quickUriGroupingRule;
/**
* Cache the HTTP URIs which are not formatted by the rules per service.
- * Level one map key is service name, the value is a map of HTTP URI and
its count.
- * Multiple matches will be counted, because it is the pattern that the
endpoint is already formatted,
- * or doesn't need to be formatted.
- * The repeatable URI is a pattern already.
+ * Level one map key is service name, the value is a map of HTTP URIs with
candidates of formatted names.
+ * If the URI is formatted by the rules, the value would be the first 10
formatted names.
+ * If the URI is unformatted, the value would be an empty queue.
*/
- private ConcurrentHashMap<String, ConcurrentHashMap<String,
AtomicInteger>> cachedHttpUris = new ConcurrentHashMap<>();
+ private ConcurrentHashMap<String, ConcurrentHashMap<String,
ArrayBlockingQueue<String>>> cachedHttpUris = new ConcurrentHashMap<>();
private final AtomicInteger aiPipelineExecutionCounter = new
AtomicInteger(0);
/**
* The max number of HTTP URIs per service for further URI pattern
recognition.
@@ -90,17 +90,26 @@ public class EndpointNameGrouping {
if (!formattedName._2() && quickUriGroupingRule != null) {
formattedName = formatByQuickUriPattern(serviceName, endpointName);
- ConcurrentHashMap<String, AtomicInteger> svrHttpUris =
cachedHttpUris.get(serviceName);
+ ConcurrentHashMap<String, ArrayBlockingQueue<String>> svrHttpUris
= cachedHttpUris.get(serviceName);
if (svrHttpUris == null) {
cachedHttpUris.putIfAbsent(serviceName, new
ConcurrentHashMap<>());
svrHttpUris = cachedHttpUris.get(serviceName);
}
// Only cache first N(determined by maxHttpUrisNumberPerService)
URIs per 30 mins.
if (svrHttpUris.size() < maxHttpUrisNumberPerService) {
- final AtomicInteger cachedCount =
svrHttpUris.putIfAbsent(formattedName._1(), new AtomicInteger(1));
- if (null != cachedCount) {
- cachedCount.incrementAndGet();
+ if (formattedName._2()) {
+ // The queue size is 10, which means only cache the first
10 formatted names.
+ final ArrayBlockingQueue<String> formattedURIs =
svrHttpUris.putIfAbsent(
+ formattedName._1(), new ArrayBlockingQueue<>(10));
+ if (formattedURIs.size() < 10) {
+ // Try to push the raw URI as a candidate of formatted
name.
+ formattedURIs.offer(endpointName);
+ }
+ } else {
+ svrHttpUris.putIfAbsent(
+ formattedName._1(), new ArrayBlockingQueue<>(0));
}
+
}
}
@@ -166,17 +175,24 @@ public class EndpointNameGrouping {
if (aiPipelineExecutionCounter.incrementAndGet()
% 30 == 0) {
// Send the cached URIs to the recognition
server per 30 mins to build new patterns.
cachedHttpUris.forEach((serviceName,
httpUris) -> {
- List<HttpUriRecognition.HTTPUri> uris
- = httpUris.keySet()
- .stream()
- .map(
- uri -> new
HttpUriRecognition.HTTPUri(
- uri,
httpUris.get(uri).get()
- ))
-
.collect(Collectors.toList());
+ final List<HttpUriRecognition.HTTPUri>
candidates4UriPatterns = new ArrayList<>(
+ 3000);
+ httpUris.forEach((uri, candidates) -> {
+ if (candidates.size() == 0) {
+ //unrecognized uri
+ candidates4UriPatterns.add(new
HttpUriRecognition.HTTPUri(uri));
+ } else {
+ String candidateUri = null;
+ while ((candidateUri =
candidates.poll()) != null) {
+ candidates4UriPatterns.add(
+ new
HttpUriRecognition.HTTPUri(candidateUri));
+ }
+ }
+ });
+
// Reset the cache once the URIs are sent
to the recognition server.
httpUris.clear();
-
httpUriRecognitionSvr.feedRawData(serviceName, uris);
+
httpUriRecognitionSvr.feedRawData(serviceName, candidates4UriPatterns);
});
} else {
// Sync with the recognition server per 1 min
to get the latest patterns.
diff --git a/test/e2e-v2/script/env b/test/e2e-v2/script/env
index ded4aab410..48ec05f718 100644
--- a/test/e2e-v2/script/env
+++ b/test/e2e-v2/script/env
@@ -21,7 +21,7 @@ SW_AGENT_GO_COMMIT=4af380c2db6243106b0fc650b6003ce3b3eb82a0
SW_AGENT_PYTHON_COMMIT=c76a6ec51a478ac91abb20ec8f22a99b8d4d6a58
SW_AGENT_CLIENT_JS_COMMIT=af0565a67d382b683c1dbd94c379b7080db61449
SW_AGENT_CLIENT_JS_TEST_COMMIT=4f1eb1dcdbde3ec4a38534bf01dded4ab5d2f016
-SW_KUBERNETES_COMMIT_SHA=b670c41d94a82ddefcf466d54bab5c492d88d772
+SW_KUBERNETES_COMMIT_SHA=e2c61c6774cf377b23516fca6f8a1e119d3191c5
SW_ROVER_COMMIT=fc8d074c6d34ecfee585a7097cbd5aef1ca680a5
SW_BANYANDB_COMMIT=dea8c1e37d4dc19fe18397deb576151a22e2fad8
