This is an automated email from the ASF dual-hosted git repository. wusheng pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push: new 3f4126a28a Refactor data-generator to support generating metrics (#12031) 3f4126a28a is described below commit 3f4126a28a2bec2ed0f7587d74b42aee1578221e Author: Gao Hongtao <hanahm...@gmail.com> AuthorDate: Sun Mar 17 23:07:03 2024 +0800 Refactor data-generator to support generating metrics (#12031) --- docs/en/changes/changes.md | 1 + docs/en/setup/backend/segment-template.json | 183 +++++++++------------ .../apache/skywalking/generator/BoolGenerator.java | 6 +- .../skywalking/generator/FixedStringGenerator.java | 6 +- .../org/apache/skywalking/generator/Generator.java | 4 +- .../apache/skywalking/generator/IntGenerator.java | 6 +- .../apache/skywalking/generator/ListGenerator.java | 6 +- .../skywalking/generator/SequenceGenerator.java | 6 +- .../skywalking/generator/StringGenerator.java | 15 +- .../apache/skywalking/generator/UUIDGenerator.java | 6 +- .../skywalking/restapi/LogGeneratorHandler.java | 2 +- .../org/apache/skywalking/restapi/LogRequest.java | 54 +++--- .../skywalking/restapi/SegmentGenerator.java | 144 ++++++++++++++++ .../restapi/SegmentGeneratorHandler.java | 38 +++-- .../apache/skywalking/restapi/SegmentRequest.java | 87 ++-------- .../apache/skywalking/restapi/SpanGenerator.java | 79 +++++---- .../apache/skywalking/restapi/TagGenerator.java | 10 +- .../src/main/resources/application.yml | 105 ++++++++++-- .../skywalking/generator/IntGeneratorTest.java | 2 +- .../generator/SequenceGeneratorTest.java | 4 +- .../SegmentGeneratorTest.java} | 34 ++-- .../src/test/resources/segment.tpl.json | 122 ++++++++++++++ 22 files changed, 615 insertions(+), 305 deletions(-) diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index b6db84afe0..ca4c1cf743 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -78,6 +78,7 @@ - VIRTUAL_MQ -> PULSAR * Add Golang as a supported language for Kafka. * Support displaying the port services listen to from OAP and UI during server start. +* Refactor data-generator to support generating metrics. #### UI diff --git a/docs/en/setup/backend/segment-template.json b/docs/en/setup/backend/segment-template.json index 88b1e5cfc6..0cfc9c1a39 100644 --- a/docs/en/setup/backend/segment-template.json +++ b/docs/en/setup/backend/segment-template.json @@ -1,123 +1,39 @@ { "traceId": { "type": "uuid", - "changingFrequency": "5" + "changingFrequency": "1" }, - "serviceInstanceName": { - "type": "randomString", - "length": "10", - "letters": true, - "numbers": true - }, - "segmentId": { - "type": "randomString", - "length": "20", - "letters": true, - "numbers": true - }, - "serviceName": { - "type": "randomString", - "length": "10", - "letters": true, - "numbers": true - }, - "endpointName": { - "type": "randomString", - "length": "10", - "prefix": "test_", - "letters": true, - "numbers": true, - "domainSize": 10 - }, - "startTime": { - "type": "sequence", - "min": "1650510807000", - "max": "1650512807000" - }, - "latency": { - "type": "randomInt", - "min": 100, - "max": 1000 - }, - "error": { - "type": "randomInt", - "min": 1, - "max": 1 - }, - "tags": { + "segments": { "type": "randomList", "size": 5, "item": { - "key": { + "serviceInstanceName": { "type": "randomString", "length": "10", - "prefix": "test_tag_", "letters": true, "numbers": true, "domainSize": 10 }, - "value": { + "serviceName": { "type": "randomString", - "length": "10", - "prefix": "test_value_", - "letters": true, - "numbers": true - } - } - }, - "spans": { - "type": "randomList", - "size": 5, - "item": { - "spanId": { - "type": "sequence", - "min": "0", - "max": "5" - }, - "parentSpanId": { - "type": "sequence", - "min": "-1", - "max": "4" - }, - "startTime": { - "type": "randomInt", - "min": "1650510807000", - "max": "1650512807000", - "domainSize": 10 - }, - "endTime": { - "type": "randomInt", - "min": "1650510807000", - "max": "1650512807000", + "prefix": "service_", + "length": "1", + "letters": false, + "numbers": true, "domainSize": 10 }, - "operationName": { + "endpointName": { "type": "randomString", "length": "10", - "prefix": "test_endpoint_", + "prefix": "test_", "letters": true, - "numbers": true - }, - "peer": { - "type": "randomString", - "length": "10", - "prefix": "test_peer_", - "letters": true, - "numbers": true - }, - "spanLayer": { - "type": "randomInt", - "min": "0", - "max": "4" - }, - "componentId": { - "type": "randomInt", - "min": "0", - "max": "4" + "numbers": true, + "domainSize": 10 }, "error": { - "type": "randomBool", - "possibility": "0.9" + "type": "randomInt", + "min": 1, + "max": 1 }, "tags": { "type": "randomList", @@ -126,20 +42,81 @@ "key": { "type": "randomString", "length": "10", - "prefix": "test_tag_key_", + "prefix": "test_tag_", "letters": true, "numbers": true, - "domainSize": 10 + "domainSize": 5 }, "value": { "type": "randomString", "length": "10", - "prefix": "test_tag_val_", + "prefix": "test_value_", + "letters": true, + "numbers": true, + "domainSize": 10 + } + } + }, + "spans": { + "type": "randomList", + "size": 5, + "item": { + "latency": { + "type": "randomInt", + "min": 100, + "max": 1000 + }, + "operationName": { + "type": "randomString", + "length": "10", + "prefix": "test_endpoint_", "letters": true, "numbers": true + }, + "peer": { + "type": "randomString", + "length": "10", + "prefix": "test_peer_", + "letters": true, + "numbers": true + }, + "spanLayer": { + "type": "randomInt", + "min": "0", + "max": "4" + }, + "componentId": { + "type": "randomInt", + "min": "0", + "max": "4" + }, + "error": { + "type": "randomBool", + "possibility": "0.2" + }, + "tags": { + "type": "randomList", + "size": 5, + "item": { + "key": { + "type": "randomString", + "length": "10", + "prefix": "test_tag_key_", + "letters": true, + "numbers": true, + "domainSize": 10 + }, + "value": { + "type": "randomString", + "length": "10", + "prefix": "test_tag_val_", + "letters": true, + "numbers": true + } + } } } } } } -} +} \ No newline at end of file diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/BoolGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/BoolGenerator.java index 7bda23aa7a..c6132de0b2 100644 --- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/BoolGenerator.java +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/BoolGenerator.java @@ -25,7 +25,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import lombok.Data; @JsonDeserialize(builder = BoolGenerator.Builder.class) -public final class BoolGenerator implements Generator<Boolean> { +public final class BoolGenerator implements Generator<Object, Boolean> { private final Random random = ThreadLocalRandom.current(); private final double possibility; @@ -34,13 +34,13 @@ public final class BoolGenerator implements Generator<Boolean> { } @Override - public Boolean next() { + public Boolean next(Object ignored) { return random.nextDouble() < possibility; } @Override public String toString() { - return String.valueOf(next()); + return String.valueOf(next(null)); } @Data diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/FixedStringGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/FixedStringGenerator.java index edf11a54ce..7d2d78ed84 100644 --- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/FixedStringGenerator.java +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/FixedStringGenerator.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import lombok.Data; @JsonDeserialize(builder = FixedStringGenerator.Builder.class) -public final class FixedStringGenerator implements Generator<String> { +public final class FixedStringGenerator implements Generator<Object, String> { private final String value; public FixedStringGenerator(Builder builder) { @@ -31,7 +31,7 @@ public final class FixedStringGenerator implements Generator<String> { } @Override - public String next() { + public String next(Object ignored) { return value; } @@ -46,6 +46,6 @@ public final class FixedStringGenerator implements Generator<String> { @Override public String toString() { - return String.valueOf(next()); + return String.valueOf(next(null)); } } diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/Generator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/Generator.java index 511f18873c..1630efccd8 100644 --- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/Generator.java +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/Generator.java @@ -36,8 +36,8 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @Type(value = SequenceGenerator.class, name = "sequence"), @Type(value = UUIDGenerator.class, name = "uuid"), }) -public interface Generator<T> { - public T next(); +public interface Generator<I, T> { + public T next(I input); default public void reset() { } diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/IntGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/IntGenerator.java index bd2b5a6e5e..e101991b3c 100644 --- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/IntGenerator.java +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/IntGenerator.java @@ -27,7 +27,7 @@ import com.google.common.base.Preconditions; import lombok.Data; @JsonDeserialize(builder = IntGenerator.Builder.class) -public final class IntGenerator implements Generator<Long> { +public final class IntGenerator implements Generator<Object, Long> { private final boolean limitedDomain; private final Long min; private final Long max; @@ -45,7 +45,7 @@ public final class IntGenerator implements Generator<Long> { } @Override - public Long next() { + public Long next(Object ignored) { if (!limitedDomain) { return next0(); } @@ -79,7 +79,7 @@ public final class IntGenerator implements Generator<Long> { @Override public String toString() { - return String.valueOf(next()); + return String.valueOf(next(null)); } @Data diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/ListGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/ListGenerator.java index 07974cd3c7..26f67d5a17 100644 --- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/ListGenerator.java +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/ListGenerator.java @@ -26,7 +26,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import lombok.Data; @JsonDeserialize(builder = ListGenerator.Builder.class) -public final class ListGenerator<T> implements Generator<List<T>> { +public final class ListGenerator<T> implements Generator<Object, List<T>> { private final T item; private final int size; @@ -36,7 +36,7 @@ public final class ListGenerator<T> implements Generator<List<T>> { } @Override - public List<T> next() { + public List<T> next(Object ignored) { return IntStream .range(0, size) .mapToObj($ -> item) @@ -45,7 +45,7 @@ public final class ListGenerator<T> implements Generator<List<T>> { @Override public void reset() { - ((Generator<?>) item).reset(); + ((Generator<?, ?>) item).reset(); } @Data diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/SequenceGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/SequenceGenerator.java index 3df3f9246a..c8ba04481e 100644 --- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/SequenceGenerator.java +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/SequenceGenerator.java @@ -29,7 +29,7 @@ import lombok.Data; import lombok.experimental.Accessors; @JsonDeserialize(builder = SequenceGenerator.Builder.class) -public final class SequenceGenerator implements Generator<Long> { +public final class SequenceGenerator implements Generator<Object, Long> { private final boolean limitedDomain; private final long min; private final long max; @@ -52,7 +52,7 @@ public final class SequenceGenerator implements Generator<Long> { } @Override - public Long next() { + public Long next(Object ignored) { if (!limitedDomain) { return next0(); } @@ -94,7 +94,7 @@ public final class SequenceGenerator implements Generator<Long> { @Override public String toString() { - return String.valueOf(next()); + return String.valueOf(next(null)); } @Data diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/StringGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/StringGenerator.java index 06a6bfb770..12e43c3691 100644 --- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/StringGenerator.java +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/StringGenerator.java @@ -29,7 +29,7 @@ import org.apache.commons.lang3.RandomStringUtils; import lombok.Data; @JsonDeserialize(builder = StringGenerator.Builder.class) -public final class StringGenerator implements Generator<String> { +public final class StringGenerator implements Generator<String, String> { private final int length; private final String prefix; private final boolean letters; @@ -58,12 +58,17 @@ public final class StringGenerator implements Generator<String> { } @Override - public String next() { + public String next(String parent) { + if (Strings.isNullOrEmpty(parent)) { + parent = ""; + } else { + parent += "-"; + } if (!limitedDomain) { - return Strings.nullToEmpty(prefix) + return parent + Strings.nullToEmpty(prefix) + RandomStringUtils.random(length, letters, numbers); } - return domain + return parent + domain .stream() .skip(random.nextInt(domain.size())) .findFirst() @@ -72,7 +77,7 @@ public final class StringGenerator implements Generator<String> { @Override public String toString() { - return String.valueOf(next()); + return String.valueOf(next(null)); } @Data diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/UUIDGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/UUIDGenerator.java index 22a75efe8b..80116c7968 100644 --- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/UUIDGenerator.java +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/UUIDGenerator.java @@ -27,7 +27,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import lombok.Data; @JsonDeserialize(builder = UUIDGenerator.Builder.class) -public final class UUIDGenerator implements Generator<String> { +public final class UUIDGenerator implements Generator<Object, String> { private final int changingFrequency; private final AtomicInteger counter; private final AtomicReference<String> last = @@ -41,7 +41,7 @@ public final class UUIDGenerator implements Generator<String> { } @Override - public String next() { + public String next(Object ignored) { if (counter.incrementAndGet() < changingFrequency) { return last.get(); } @@ -51,7 +51,7 @@ public final class UUIDGenerator implements Generator<String> { @Override public String toString() { - return String.valueOf(next()); + return String.valueOf(next(null)); } @Override diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/LogGeneratorHandler.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/LogGeneratorHandler.java index 303bed64f2..bc096e9862 100644 --- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/LogGeneratorHandler.java +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/LogGeneratorHandler.java @@ -70,7 +70,7 @@ public class LogGeneratorHandler { log.info("Generate {} mock logs, qps: {}, template: {}", size, qps, request); final IntConsumer generator = unused -> { - final Log l = request.next(); + final Log l = request.next(null); log.debug("Generating log: {}", l); sourceReceiver.receive(l); }; diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/LogRequest.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/LogRequest.java index bf41e2bf49..c8fbf0c29a 100644 --- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/LogRequest.java +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/LogRequest.java @@ -38,51 +38,55 @@ import lombok.SneakyThrows; @Data @JsonTypeInfo(use = JsonTypeInfo.Id.NONE) -public final class LogRequest implements Generator<Log> { +public final class LogRequest implements Generator<Object, Log> { @JsonIgnore private final ObjectMapper om = new ObjectMapper(); - private Generator<Long> timestamp; - private Generator<String> serviceName; - private Generator<String> serviceInstanceName; - private Generator<String> endpointName; - private Generator<String> traceId; - private Generator<String> traceSegmentId; - private Generator<Long> spanId; - private Generator<Long> contentType; - private Generator<String> content; - private Generator<List<TagGenerator>> tags; - private Generator<Boolean> error; + private Generator<Object, Long> timestamp; + private Generator<Object, String> serviceName; + private Generator<Object, String> serviceInstanceName; + private Generator<Object, String> endpointName; + private Generator<Object, String> traceId; + private Generator<Object, String> traceSegmentId; + private Generator<Object, Long> spanId; + private Generator<Object, Long> contentType; + private Generator<Object, String> content; + private Generator<Object, List<TagGenerator>> tags; + private Generator<Object, Boolean> error; @SneakyThrows @Override - public Log next() { + public Log next(Object ignored) { final Log log = new Log(); - log.setTimestamp(getTimestamp().next()); + if (timestamp == null) { + log.setTimestamp(System.currentTimeMillis()); + } else { + log.setTimestamp(getTimestamp().next(null)); + } log.setServiceId( IDManager.ServiceID.buildId( - getServiceName().next(), + getServiceName().next(null), true)); log.setServiceInstanceId( IDManager.ServiceInstanceID.buildId( log.getServiceId(), - getServiceInstanceName().next())); + getServiceInstanceName().next(null))); log.setEndpointId( IDManager.EndpointID.buildId( log.getServiceId(), - getEndpointName().next())); - log.setTraceId(getTraceId().next()); - log.setTraceSegmentId(getTraceSegmentId().next()); - log.setSpanId(getSpanId().next().intValue()); - log.setContentType(ContentType.instanceOf(getContentType().next().intValue())); - log.setContent(getContent().next()); - log.setError(getError().next()); + getEndpointName().next(null))); + log.setTraceId(getTraceId().next(null)); + log.setTraceSegmentId(getTraceSegmentId().next(null)); + log.setSpanId(getSpanId().next(null).intValue()); + log.setContentType(ContentType.instanceOf(getContentType().next(null).intValue())); + log.setContent(getContent().next(null)); + log.setError(getError().next(null)); log.setTimeBucket(TimeBucket.getRecordTimeBucket(log.getTimestamp())); log.setTags( getTags() - .next() + .next(null) .stream() - .map(TagGenerator::next) + .map(tg -> tg.next(null)) .collect(Collectors.<Tag>toList())); log.setTagsRawData( LogTags diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGenerator.java new file mode 100644 index 0000000000..899a3c164e --- /dev/null +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGenerator.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.restapi; + +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.common.base.Strings; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; +import org.apache.skywalking.apm.network.language.agent.v3.SegmentReference; +import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; +import org.apache.skywalking.generator.Generator; +import org.apache.skywalking.generator.StringGenerator; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; +import org.apache.skywalking.oap.server.core.source.Segment; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +@Getter +@Setter +@JsonTypeInfo(use = JsonTypeInfo.Id.NONE) +public class SegmentGenerator implements Generator<SegmentGenerator.SegmentContext, SegmentGenerator.SegmentResult> { + + private Generator<String, String> segmentId; + private Generator<String, String> serviceName; + private Generator<String, String> serviceInstanceName; + private Generator<String, String> endpointName; + private Generator<Object, Long> error; + private Generator<Object, List<TagGenerator>> tags; + private Generator<Object, List<SpanGenerator>> spans; + + @Override + public SegmentResult next(SegmentContext ctx) { + long now = System.currentTimeMillis(); + final String serviceName = getServiceName().next(null); + final String serviceInstanceName = getServiceInstanceName().next(serviceName); + final String endpointName = getEndpointName().next(null); + if (segmentId == null) { + StringGenerator.Builder segmentIdBuilder = new StringGenerator.Builder(); + segmentIdBuilder.setLength(20); + segmentIdBuilder.setNumbers(true); + segmentIdBuilder.setLetters(true); + segmentId = segmentIdBuilder.build(); + } + + final SegmentReference sr = Optional.ofNullable(ctx.parentSegment).flatMap(parentSegment -> parentSegment.segmentObject.getSpansList().stream() + .filter(span -> !Strings.isNullOrEmpty(span.getPeer())) + .findFirst().map(span -> SegmentReference + .newBuilder() + .setTraceId(ctx.traceId) + .setParentServiceInstance(parentSegment.segmentObject.getServiceInstance()) + .setParentService(parentSegment.segmentObject.getService()) + .setParentSpanId(span.getSpanId()) + .setParentTraceSegmentId(parentSegment.segment.getSegmentId()) + .setNetworkAddressUsedAtPeer(span.getPeer()) + .build())) + .orElse(null); + final String segmentId = getSegmentId().next(null); + + final List<SpanGenerator> spanGenerators = getSpans().next(null); + int size = spanGenerators.size(); + final SegmentObject segmentObj = SegmentObject + .newBuilder() + .setTraceId(ctx.traceId) + .setTraceSegmentId(segmentId) + .addAllSpans( + IntStream.range(0, size) + .mapToObj(i -> { + SpanGenerator sg = spanGenerators.get(i); + return sg.next(new SpanGenerator.SpanGeneratorContext(i, size, sr)); + }) + .collect(Collectors.<SpanObject>toList())) + .setService(serviceName) + .setServiceInstance(serviceInstanceName) + .build(); + + // Reset the span generator to generate the span id from 0 + getSpans().reset(); + + Long latency = segmentObj.getSpansList().stream().reduce(0L, (l, span) -> l + (span.getEndTime() - span.getStartTime()), Long::sum); + + final Segment segment = new Segment(); + segment.setSegmentId(segmentId); + segment.setTraceId(ctx.traceId); + segment.setServiceId( + IDManager.ServiceID.buildId(serviceName, true)); + segment.setServiceInstanceId( + IDManager.ServiceInstanceID.buildId( + segment.getServiceId(), + serviceInstanceName)); + segment.setEndpointId( + IDManager.EndpointID.buildId( + segment.getServiceId(), + endpointName)); + segment.setStartTime(now - latency); + segment.setLatency(latency.intValue()); + segment.setIsError(getError().next(null).intValue()); + segment.setTimeBucket(TimeBucket.getRecordTimeBucket(segment.getStartTime())); + segment.setTags( + getTags() + .next(null) + .stream() + .map(tg -> tg.next(null)) + .collect(Collectors.<Tag>toList())); + + segment.setDataBinary(segmentObj.toByteArray()); + return new SegmentResult(segment, segmentObj); + } + + @RequiredArgsConstructor + public static class SegmentContext { + final String traceId; + final SegmentResult parentSegment; + } + + @RequiredArgsConstructor + public static class SegmentResult { + final Segment segment; + final SegmentObject segmentObject; + } +} diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGeneratorHandler.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGeneratorHandler.java index fb5a0db01e..6e7a520dbf 100644 --- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGeneratorHandler.java +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGeneratorHandler.java @@ -19,14 +19,6 @@ package org.apache.skywalking.restapi; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.CancellationException; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; -import java.util.function.IntConsumer; -import java.util.stream.IntStream; import com.linecorp.armeria.common.HttpResponse; import com.linecorp.armeria.common.HttpStatus; import com.linecorp.armeria.common.MediaType; @@ -38,22 +30,35 @@ import com.linecorp.armeria.server.annotation.Param; import com.linecorp.armeria.server.annotation.Post; import com.linecorp.armeria.server.annotation.ProducesJson; import com.linecorp.armeria.server.annotation.RequestObject; -import org.apache.skywalking.oap.server.core.CoreModule; -import org.apache.skywalking.oap.server.core.source.Segment; -import org.apache.skywalking.oap.server.core.source.SourceReceiver; -import org.apache.skywalking.oap.server.library.module.ModuleManager; import io.netty.channel.EventLoopGroup; import io.netty.util.concurrent.Future; import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.analyzer.module.AnalyzerModule; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.ISegmentParserService; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.library.module.ModuleManager; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.function.IntConsumer; +import java.util.stream.IntStream; @Slf4j public class SegmentGeneratorHandler { private final SourceReceiver sourceReceiver; private final Map<String, Future<?>> futures = new ConcurrentHashMap<>(); private final EventLoopGroup eventLoopGroup = EventLoopGroups.newEventLoopGroup(10); + private final ISegmentParserService segmentParserService; public SegmentGeneratorHandler(ModuleManager manager) { sourceReceiver = manager.find(CoreModule.NAME).provider().getService(SourceReceiver.class); + segmentParserService = manager.find(AnalyzerModule.NAME).provider().getService(ISegmentParserService.class); } @Post("/mock-data/segments/tasks") @@ -71,9 +76,12 @@ public class SegmentGeneratorHandler { log.info("Generate {} mock segments, qps: {}, template: {}", size, qps, request); final IntConsumer generator = unused -> { - final Segment segment = request.next(); - log.debug("Generating segment: {}", segment); - sourceReceiver.receive(segment); + final List<SegmentGenerator.SegmentResult> segments = request.next(null); + log.debug("Generating segment: {}", (Object) segments); + segments.forEach(s -> { + sourceReceiver.receive(s.segment); + segmentParserService.send(s.segmentObject); + }); }; final String requestId = UUID.randomUUID().toString(); final Future<?> future; diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentRequest.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentRequest.java index 70d4d63167..8fa57d30ef 100644 --- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentRequest.java +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentRequest.java @@ -19,82 +19,29 @@ package org.apache.skywalking.restapi; -import java.util.List; -import java.util.stream.Collectors; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; -import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; -import org.apache.skywalking.generator.Generator; -import org.apache.skywalking.oap.server.core.analysis.IDManager; -import org.apache.skywalking.oap.server.core.analysis.TimeBucket; -import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; -import org.apache.skywalking.oap.server.core.source.Segment; import lombok.Data; +import org.apache.skywalking.generator.Generator; + +import java.util.ArrayList; +import java.util.List; @Data @JsonTypeInfo(use = JsonTypeInfo.Id.NONE) -public final class SegmentRequest implements Generator<Segment> { - private Generator<String> segmentId; - private Generator<String> traceId; - private Generator<String> serviceName; - private Generator<String> serviceInstanceName; - private Generator<String> endpointName; - private Generator<Long> startTime; - private Generator<Long> latency; - private Generator<Long> error; - private Generator<List<TagGenerator>> tags; - private Generator<List<SpanGenerator>> spans; +public final class SegmentRequest implements Generator<Object, List<SegmentGenerator.SegmentResult>> { + private Generator<String, String> traceId; + private Generator<Object, List<SegmentGenerator>> segments; @Override - public Segment next() { - final String traceId = getTraceId().next(); - final String serviceName = getServiceName().next(); - final String serviceInstanceName = getServiceInstanceName().next(); - final String endpointName = getEndpointName().next(); - final String segmentId = getSegmentId().next(); - - final SegmentObject segmentObj = SegmentObject - .newBuilder() - .setTraceId(traceId) - .setTraceSegmentId(segmentId) - .addAllSpans( - getSpans() - .next() - .stream() - .map(SpanGenerator::next) - .collect(Collectors.<SpanObject>toList())) - .setService(serviceName) - .setServiceInstance(serviceInstanceName) - .build(); - - // Reset the span generator to generate the span id from 0 - getSpans().reset(); - - final Segment segment = new Segment(); - segment.setSegmentId(segmentId); - segment.setTraceId(traceId); - segment.setServiceId( - IDManager.ServiceID.buildId(serviceName, true)); - segment.setServiceInstanceId( - IDManager.ServiceInstanceID.buildId( - segment.getServiceId(), - serviceInstanceName)); - segment.setEndpointId( - IDManager.EndpointID.buildId( - segment.getServiceId(), - endpointName)); - segment.setStartTime(getStartTime().next()); - segment.setLatency(getLatency().next().intValue()); - segment.setIsError(getError().next().intValue()); - segment.setTimeBucket(TimeBucket.getRecordTimeBucket(segment.getStartTime())); - segment.setTags( - getTags() - .next() - .stream() - .map(TagGenerator::next) - .collect(Collectors.<Tag>toList())); - - segment.setDataBinary(segmentObj.toByteArray()); - return segment; + public List<SegmentGenerator.SegmentResult> next(Object ignored) { + final String traceId = getTraceId().next(null); + final List<SegmentGenerator> segments = getSegments().next(traceId); + SegmentGenerator.SegmentResult last = null; + List<SegmentGenerator.SegmentResult> result = new ArrayList<>(segments.size()); + for (SegmentGenerator each : segments) { + last = each.next(new SegmentGenerator.SegmentContext(traceId, last)); + result.add(last); + } + return result; } } diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SpanGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SpanGenerator.java index b8af9aea77..708e58fe0b 100644 --- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SpanGenerator.java +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SpanGenerator.java @@ -22,9 +22,12 @@ package org.apache.skywalking.restapi; import java.util.List; import java.util.stream.Collectors; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import lombok.RequiredArgsConstructor; import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; +import org.apache.skywalking.apm.network.language.agent.v3.SegmentReference; import org.apache.skywalking.apm.network.language.agent.v3.SpanLayer; import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; +import org.apache.skywalking.apm.network.language.agent.v3.SpanType; import org.apache.skywalking.generator.Generator; import lombok.Getter; import lombok.Setter; @@ -32,49 +35,58 @@ import lombok.Setter; @Getter @Setter @JsonTypeInfo(use = JsonTypeInfo.Id.NONE) -public final class SpanGenerator implements Generator<SpanObject> { - private Generator<Long> spanId; - private Generator<Long> parentSpanId; - private Generator<Long> startTime; - private Generator<Long> endTime; - private Generator<String> operationName; - private Generator<String> peer; - private Generator<Long> spanLayer; - private Generator<Long> componentId; - private Generator<Boolean> error; - private Generator<List<TagGenerator>> tags; +public final class SpanGenerator implements Generator<SpanGenerator.SpanGeneratorContext, SpanObject> { + private Generator<Object, Long> endTime; + private Generator<Object, Long> latency; + private Generator<String, String> operationName; + private Generator<String, String> peer; + private Generator<Object, Long> spanLayer; + private Generator<Object, Long> componentId; + private Generator<Object, Boolean> error; + private Generator<Object, List<TagGenerator>> tags; @Override - public SpanObject next() { - return SpanObject + public SpanObject next(SpanGenerator.SpanGeneratorContext ctx) { + long now; + if (endTime == null) { + now = System.currentTimeMillis(); + } else { + now = endTime.next(null); + } + SpanObject.Builder sob = SpanObject .newBuilder() - .setSpanId(getSpanId().next().intValue()) - .setParentSpanId(getParentSpanId().next().intValue()) - .setStartTime(getStartTime().next()) - .setEndTime(getEndTime().next()) - .setOperationName(getOperationName().next()) - .setPeer(getPeer().next()) - .setSpanLayer(SpanLayer.forNumber(getSpanLayer().next().intValue())) - .setComponentId(getComponentId().next().intValue()) - .setIsError(getError().next()) + .setSpanId(ctx.index) + .setParentSpanId(ctx.index - 1) + .setStartTime(now - latency.next(null)) + .setEndTime(now) + .setComponentId(getComponentId().next(null).intValue()) + .setIsError(getError().next(null)) .addAllTags( getTags() - .next() + .next(null) .stream() - .map(TagGenerator::next) + .map(tg -> tg.next(null)) .map(it -> KeyStringValuePair .newBuilder().setKey(it.getKey()) .setValue(it.getValue()).build()) - .collect(Collectors.toList())) - .build(); + .collect(Collectors.toList())); + if (ctx.index == 0) { + sob.setSpanLayer(SpanLayer.forNumber(getSpanLayer().next(null).intValue())) + .setSpanType(SpanType.Entry); + if (ctx.ref != null) { + sob.addRefs(ctx.ref); + } + } else if (ctx.length > 1 && ctx.index == ctx.length - 1) { + sob.setSpanType(SpanType.Exit).setPeer(getPeer().next(null)).setSpanLayer(SpanLayer.Database); + } else { + sob.setSpanLayer(SpanLayer.forNumber(getSpanLayer().next(null).intValue())) + .setSpanType(SpanType.Local); + } + return sob.build(); } @Override public void reset() { - spanId.reset(); - parentSpanId.reset(); - startTime.reset(); - endTime.reset(); operationName.reset(); peer.reset(); spanLayer.reset(); @@ -82,4 +94,11 @@ public final class SpanGenerator implements Generator<SpanObject> { error.reset(); tags.reset(); } + + @RequiredArgsConstructor + public static class SpanGeneratorContext { + final int index; + final int length; + final SegmentReference ref; + } } diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/TagGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/TagGenerator.java index e55d79ba2d..74a6b5d5f7 100644 --- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/TagGenerator.java +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/TagGenerator.java @@ -28,13 +28,13 @@ import lombok.Setter; @Getter @Setter @JsonTypeInfo(use = JsonTypeInfo.Id.NONE) -public final class TagGenerator implements Generator<Tag> { - private Generator<String> key; - private Generator<String> value; +public final class TagGenerator implements Generator<Object, Tag> { + private Generator<String, String> key; + private Generator<String, String> value; @Override - public Tag next() { - return new Tag(key.next(), value.next()); + public Tag next(Object ignored) { + return new Tag(key.next(null), value.next(null)); } @Override diff --git a/oap-server/server-tools/data-generator/src/main/resources/application.yml b/oap-server/server-tools/data-generator/src/main/resources/application.yml index 083a10d141..62b0ddc85a 100755 --- a/oap-server/server-tools/data-generator/src/main/resources/application.yml +++ b/oap-server/server-tools/data-generator/src/main/resources/application.yml @@ -53,9 +53,6 @@ core: storageSessionTimeout: ${SW_CORE_STORAGE_SESSION_TIMEOUT:70000} # The period of doing data persistence. Unit is second.Default value is 25s persistentPeriod: ${SW_CORE_PERSISTENT_PERIOD:25} - # Cache metrics data for 1 minute to reduce database queries, and if the OAP cluster changes within that minute, - # the metrics may not be accurate within that minute. - enableDatabaseSession: ${SW_CORE_ENABLE_DATABASE_SESSION:true} topNReportPeriod: ${SW_CORE_TOPN_REPORT_PERIOD:10} # top_n record worker report cycle, unit is minute # Extra model column are the column defined by in the codes, These columns of model are not required logically in aggregation or further query, # and it will cause more load for memory, network of OAP and storage. @@ -63,19 +60,34 @@ core: activeExtraModelColumns: ${SW_CORE_ACTIVE_EXTRA_MODEL_COLUMNS:false} # The max length of service + instance names should be less than 200 serviceNameMaxLength: ${SW_SERVICE_NAME_MAX_LENGTH:70} + # The period(in seconds) of refreshing the service cache. Default value is 10s. + serviceCacheRefreshInterval: ${SW_SERVICE_CACHE_REFRESH_INTERVAL:10} instanceNameMaxLength: ${SW_INSTANCE_NAME_MAX_LENGTH:70} # The max length of service + endpoint names should be less than 240 endpointNameMaxLength: ${SW_ENDPOINT_NAME_MAX_LENGTH:150} # Define the set of span tag keys, which should be searchable through the GraphQL. - searchableTracesTags: ${SW_SEARCHABLE_TAG_KEYS:http.method,status_code,db.type,db.instance,mq.queue,mq.topic,mq.broker} + # The max length of key=value should be less than 256 or will be dropped. + searchableTracesTags: ${SW_SEARCHABLE_TAG_KEYS:http.method,http.status_code,rpc.status_code,db.type,db.instance,mq.queue,mq.topic,mq.broker} # Define the set of log tag keys, which should be searchable through the GraphQL. - searchableLogsTags: ${SW_SEARCHABLE_LOGS_TAG_KEYS:level} + # The max length of key=value should be less than 256 or will be dropped. + searchableLogsTags: ${SW_SEARCHABLE_LOGS_TAG_KEYS:level,http.status_code} # Define the set of alarm tag keys, which should be searchable through the GraphQL. + # The max length of key=value should be less than 256 or will be dropped. searchableAlarmTags: ${SW_SEARCHABLE_ALARM_TAG_KEYS:level} + # The max size of tags keys for autocomplete select. + autocompleteTagKeysQueryMaxSize: ${SW_AUTOCOMPLETE_TAG_KEYS_QUERY_MAX_SIZE:100} + # The max size of tags values for autocomplete select. + autocompleteTagValuesQueryMaxSize: ${SW_AUTOCOMPLETE_TAG_VALUES_QUERY_MAX_SIZE:100} # The number of threads used to prepare metrics data to the storage. prepareThreads: ${SW_CORE_PREPARE_THREADS:2} # Turn it on then automatically grouping endpoint by the given OpenAPI definitions. enableEndpointNameGroupingByOpenapi: ${SW_CORE_ENABLE_ENDPOINT_NAME_GROUPING_BY_OPENAPI:true} + # The period of HTTP URI pattern recognition. Unit is second. + syncPeriodHttpUriRecognitionPattern: ${SW_CORE_SYNC_PERIOD_HTTP_URI_RECOGNITION_PATTERN:10} + # The training period of HTTP URI pattern recognition. Unit is second. + trainingPeriodHttpUriRecognitionPattern: ${SW_CORE_TRAINING_PERIOD_HTTP_URI_RECOGNITION_PATTERN:60} + # The max number of HTTP URIs per service for further URI pattern recognition. + maxHttpUrisNumberPerService: ${SW_CORE_MAX_HTTP_URIS_NUMBER_PER_SVR:3000} storage: selector: ${SW_STORAGE:h2} elasticsearch: @@ -94,32 +106,77 @@ storage: dayStep: ${SW_STORAGE_DAY_STEP:1} # Represent the number of days in the one minute/hour/day index. indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:1} # Shard number of new indexes indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:1} # Replicas number of new indexes + # Specify the settings for each index individually. + # If configured, this setting has the highest priority and overrides the generic settings. + specificIndexSettings: ${SW_STORAGE_ES_SPECIFIC_INDEX_SETTINGS:""} # Super data set has been defined in the codes, such as trace segments.The following 3 config would be improve es performance when storage super size data in es. superDatasetDayStep: ${SW_STORAGE_ES_SUPER_DATASET_DAY_STEP:-1} # Represent the number of days in the super size dataset record index, the default value is the same as dayStep when the value is less than 0 - superDatasetIndexShardsFactor: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR:5} # This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces. + superDatasetIndexShardsFactor: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR:5} # This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin traces. superDatasetIndexReplicasNumber: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_REPLICAS_NUMBER:0} # Represent the replicas number in the super size dataset record index, the default value is 0. indexTemplateOrder: ${SW_STORAGE_ES_INDEX_TEMPLATE_ORDER:0} # the order of index template bulkActions: ${SW_STORAGE_ES_BULK_ACTIONS:5000} # Execute the async bulk record data every ${SW_STORAGE_ES_BULK_ACTIONS} requests - # flush the bulk every 10 seconds whatever the number of requests - # INT(flushInterval * 2/3) would be used for index refresh period. - flushInterval: ${SW_STORAGE_ES_FLUSH_INTERVAL:15} + batchOfBytes: ${SW_STORAGE_ES_BATCH_OF_BYTES:10485760} # A threshold to control the max body size of ElasticSearch Bulk flush. + # flush the bulk every 5 seconds whatever the number of requests + flushInterval: ${SW_STORAGE_ES_FLUSH_INTERVAL:5} concurrentRequests: ${SW_STORAGE_ES_CONCURRENT_REQUESTS:2} # the number of concurrent requests resultWindowMaxSize: ${SW_STORAGE_ES_QUERY_MAX_WINDOW_SIZE:10000} - metadataQueryMaxSize: ${SW_STORAGE_ES_QUERY_MAX_SIZE:5000} + metadataQueryMaxSize: ${SW_STORAGE_ES_QUERY_MAX_SIZE:10000} + scrollingBatchSize: ${SW_STORAGE_ES_SCROLLING_BATCH_SIZE:5000} segmentQueryMaxSize: ${SW_STORAGE_ES_QUERY_SEGMENT_SIZE:200} profileTaskQueryMaxSize: ${SW_STORAGE_ES_QUERY_PROFILE_TASK_SIZE:200} + profileDataQueryBatchSize: ${SW_STORAGE_ES_QUERY_PROFILE_DATA_BATCH_SIZE:100} oapAnalyzer: ${SW_STORAGE_ES_OAP_ANALYZER:"{\"analyzer\":{\"oap_analyzer\":{\"type\":\"stop\"}}}"} # the oap analyzer. oapLogAnalyzer: ${SW_STORAGE_ES_OAP_LOG_ANALYZER:"{\"analyzer\":{\"oap_log_analyzer\":{\"type\":\"standard\"}}}"} # the oap log analyzer. It could be customized by the ES analyzer configuration to support more language log formats, such as Chinese log, Japanese log and etc. advanced: ${SW_STORAGE_ES_ADVANCED:""} + # Enable shard metrics and records indices into multi-physical indices, one index template per metric/meter aggregation function or record. + logicSharding: ${SW_STORAGE_ES_LOGIC_SHARDING:false} + # Custom routing can reduce the impact of searches. Instead of having to fan out a search request to all the shards in an index, the request can be sent to just the shard that matches the specific routing value (or values). + enableCustomRouting: ${SW_STORAGE_ES_ENABLE_CUSTOM_ROUTING:false} h2: properties: jdbcUrl: ${SW_STORAGE_H2_URL:jdbc:h2:mem:skywalking-oap-db;DB_CLOSE_DELAY=-1;DATABASE_TO_UPPER=FALSE} dataSource.user: ${SW_STORAGE_H2_USER:sa} metadataQueryMaxSize: ${SW_STORAGE_H2_QUERY_MAX_SIZE:5000} - maxSizeOfArrayColumn: ${SW_STORAGE_MAX_SIZE_OF_ARRAY_COLUMN:20} - numOfSearchableValuesPerTag: ${SW_STORAGE_NUM_OF_SEARCHABLE_VALUES_PER_TAG:2} maxSizeOfBatchSql: ${SW_STORAGE_MAX_SIZE_OF_BATCH_SQL:100} asyncBatchPersistentPoolSize: ${SW_STORAGE_ASYNC_BATCH_PERSISTENT_POOL_SIZE:1} + mysql: + properties: + jdbcUrl: ${SW_JDBC_URL:"jdbc:mysql://localhost:3306/swtest?rewriteBatchedStatements=true&allowMultiQueries=true"} + dataSource.user: ${SW_DATA_SOURCE_USER:root} + dataSource.password: ${SW_DATA_SOURCE_PASSWORD:root@1234} + dataSource.cachePrepStmts: ${SW_DATA_SOURCE_CACHE_PREP_STMTS:true} + dataSource.prepStmtCacheSize: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_SIZE:250} + dataSource.prepStmtCacheSqlLimit: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_LIMIT:2048} + dataSource.useServerPrepStmts: ${SW_DATA_SOURCE_USE_SERVER_PREP_STMTS:true} + metadataQueryMaxSize: ${SW_STORAGE_MYSQL_QUERY_MAX_SIZE:5000} + maxSizeOfBatchSql: ${SW_STORAGE_MAX_SIZE_OF_BATCH_SQL:2000} + asyncBatchPersistentPoolSize: ${SW_STORAGE_ASYNC_BATCH_PERSISTENT_POOL_SIZE:4} + postgresql: + properties: + jdbcUrl: ${SW_JDBC_URL:"jdbc:postgresql://localhost:5432/skywalking"} + dataSource.user: ${SW_DATA_SOURCE_USER:postgres} + dataSource.password: ${SW_DATA_SOURCE_PASSWORD:123456} + dataSource.cachePrepStmts: ${SW_DATA_SOURCE_CACHE_PREP_STMTS:true} + dataSource.prepStmtCacheSize: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_SIZE:250} + dataSource.prepStmtCacheSqlLimit: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_LIMIT:2048} + dataSource.useServerPrepStmts: ${SW_DATA_SOURCE_USE_SERVER_PREP_STMTS:true} + metadataQueryMaxSize: ${SW_STORAGE_MYSQL_QUERY_MAX_SIZE:5000} + maxSizeOfBatchSql: ${SW_STORAGE_MAX_SIZE_OF_BATCH_SQL:2000} + asyncBatchPersistentPoolSize: ${SW_STORAGE_ASYNC_BATCH_PERSISTENT_POOL_SIZE:4} + banyandb: + targets: ${SW_STORAGE_BANYANDB_TARGETS:127.0.0.1:17912} + maxBulkSize: ${SW_STORAGE_BANYANDB_MAX_BULK_SIZE:5000} + flushInterval: ${SW_STORAGE_BANYANDB_FLUSH_INTERVAL:15} + metricsShardsNumber: ${SW_STORAGE_BANYANDB_METRICS_SHARDS_NUMBER:1} + recordShardsNumber: ${SW_STORAGE_BANYANDB_RECORD_SHARDS_NUMBER:1} + superDatasetShardsFactor: ${SW_STORAGE_BANYANDB_SUPERDATASET_SHARDS_FACTOR:2} + concurrentWriteThreads: ${SW_STORAGE_BANYANDB_CONCURRENT_WRITE_THREADS:15} + profileTaskQueryMaxSize: ${SW_STORAGE_BANYANDB_PROFILE_TASK_QUERY_MAX_SIZE:200} # the max number of fetch task in a request + blockIntervalHours: ${SW_STORAGE_BANYANDB_BLOCK_INTERVAL_HOURS:24} # Unit is hour + segmentIntervalDays: ${SW_STORAGE_BANYANDB_SEGMENT_INTERVAL_DAYS:1} # Unit is day + superDatasetBlockIntervalHours: ${SW_STORAGE_BANYANDB_SUPER_DATASET_BLOCK_INTERVAL_HOURS:4} # Unit is hour + superDatasetSegmentIntervalDays: ${SW_STORAGE_BANYANDB_SUPER_DATASET_SEGMENT_INTERVAL_DAYS:1} # Unit is day + specificGroupSettings: ${SW_STORAGE_BANYANDB_SPECIFIC_GROUP_SETTINGS:""} # For example, {"group1": {"blockIntervalHours": 4, "segmentIntervalDays": 1}} agent-analyzer: selector: ${SW_AGENT_ANALYZER:default} @@ -132,13 +189,15 @@ agent-analyzer: # Nginx and Envoy agents can't get the real remote address. # Exit spans with the component in the list would not generate the client-side instance relation metrics. noUpstreamRealAddressAgents: ${SW_NO_UPSTREAM_REAL_ADDRESS:6000,9000} - meterAnalyzerActiveFiles: ${SW_METER_ANALYZER_ACTIVE_FILES:datasource,threadpool} # Which files could be meter analyzed, files split by "," + meterAnalyzerActiveFiles: ${SW_METER_ANALYZER_ACTIVE_FILES:datasource,threadpool,satellite,go-runtime,python-runtime,continuous-profiling} # Which files could be meter analyzed, files split by "," + slowCacheReadThreshold: ${SW_SLOW_CACHE_SLOW_READ_THRESHOLD:default:20,redis:10} # The slow cache read operation thresholds. Unit ms. + slowCacheWriteThreshold: ${SW_SLOW_CACHE_SLOW_WRITE_THRESHOLD:default:20,redis:10} # The slow cache write operation thresholds. Unit ms. log-analyzer: selector: ${SW_LOG_ANALYZER:default} default: - lalFiles: ${SW_LOG_LAL_FILES:default} - malFiles: ${SW_LOG_MAL_FILES:""} + lalFiles: ${SW_LOG_LAL_FILES:envoy-als,mesh-dp,mysql-slowsql,pgsql-slowsql,redis-slowsql,k8s-service,nginx,default} + malFiles: ${SW_LOG_MAL_FILES:"nginx"} event-analyzer: selector: ${SW_EVENT_ANALYZER:default} @@ -147,12 +206,20 @@ event-analyzer: query: selector: ${SW_QUERY:graphql} graphql: - path: ${SW_QUERY_GRAPHQL_PATH:/graphql} # Enable the log testing API to test the LAL. # NOTE: This API evaluates untrusted code on the OAP server. # A malicious script can do significant damage (steal keys and secrets, remove files and directories, install malware, etc). # As such, please enable this API only when you completely trust your users. enableLogTestTool: ${SW_QUERY_GRAPHQL_ENABLE_LOG_TEST_TOOL:false} + # Maximum complexity allowed for the GraphQL query that can be used to + # abort a query if the total number of data fields queried exceeds the defined threshold. + maxQueryComplexity: ${SW_QUERY_MAX_QUERY_COMPLEXITY:3000} + # Allow user add, disable and update UI template + enableUpdateUITemplate: ${SW_ENABLE_UPDATE_UI_TEMPLATE:false} + # "On demand log" allows users to fetch Pod containers' log in real time, + # because this might expose secrets in the logs (if any), users need + # to enable this manually, and add permissions to OAP cluster role. + enableOnDemandPodLog: ${SW_ENABLE_ON_DEMAND_POD_LOG:false} alarm: selector: ${SW_ALARM:default} @@ -171,6 +238,12 @@ health-checker: default: checkIntervalSeconds: ${SW_HEALTH_CHECKER_INTERVAL_SECONDS:5} +ai-pipeline: + selector: ${SW_AI_PIPELINE:default} + default: + uriRecognitionServerAddr: ${SW_AI_PIPELINE_URI_RECOGNITION_SERVER_ADDR:} + uriRecognitionServerPort: ${SW_AI_PIPELINE_URI_RECOGNITION_SERVER_PORT:17128} + data-generator: selector: ${SW_DATA_GENERATOR:default} default: diff --git a/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/IntGeneratorTest.java b/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/IntGeneratorTest.java index d53b007414..8098ea8092 100644 --- a/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/IntGeneratorTest.java +++ b/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/IntGeneratorTest.java @@ -30,7 +30,7 @@ public class IntGeneratorTest { builder.setMax(1L); final IntGenerator generator = builder.build(); for (int i = 0; i < 100; i++) { - assertEquals(1, generator.next().intValue()); + assertEquals(1, generator.next(null).intValue()); } } } diff --git a/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/SequenceGeneratorTest.java b/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/SequenceGeneratorTest.java index b875aa0d21..3821ad7fd1 100644 --- a/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/SequenceGeneratorTest.java +++ b/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/SequenceGeneratorTest.java @@ -33,7 +33,7 @@ public final class SequenceGeneratorTest { .build(); for (int i = 0; i < 10; i++) { - assertEquals(i + 1, generator.next().intValue()); + assertEquals(i + 1, generator.next(null).intValue()); } } @@ -47,7 +47,7 @@ public final class SequenceGeneratorTest { .build(); for (int i = 1; i < 10; i++) { - Long next = generator.next(); + Long next = generator.next(null); assertTrue(i <= next.intValue()); assertTrue(i * 2 >= next.intValue()); } diff --git a/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/IntGeneratorTest.java b/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/restapi/SegmentGeneratorTest.java similarity index 51% copy from oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/IntGeneratorTest.java copy to oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/restapi/SegmentGeneratorTest.java index d53b007414..29c9228fdb 100644 --- a/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/IntGeneratorTest.java +++ b/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/restapi/SegmentGeneratorTest.java @@ -17,20 +17,30 @@ * under the License. */ - package org.apache.skywalking.generator; +package org.apache.skywalking.restapi; -import static org.junit.jupiter.api.Assertions.assertEquals; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; -public class IntGeneratorTest { +import java.io.File; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.List; + +import static graphql.Assert.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +class SegmentGeneratorTest { + @Test - public void testFixedInt() { - final IntGenerator.Builder builder = new IntGenerator.Builder(); - builder.setMin(1L); - builder.setMax(1L); - final IntGenerator generator = builder.build(); - for (int i = 0; i < 100; i++) { - assertEquals(1, generator.next().intValue()); - } + void next() throws URISyntaxException, IOException { + ObjectMapper objectMapper = new ObjectMapper(); + URL url = getClass().getClassLoader().getResource("segment.tpl.json"); + assertNotNull(url); + File jsonFile = new File(url.toURI()); + SegmentRequest sr = objectMapper.readValue(jsonFile, SegmentRequest.class); + List<SegmentGenerator.SegmentResult> ss = sr.next(null); + assertFalse(ss.isEmpty()); } -} +} \ No newline at end of file diff --git a/oap-server/server-tools/data-generator/src/test/resources/segment.tpl.json b/oap-server/server-tools/data-generator/src/test/resources/segment.tpl.json new file mode 100644 index 0000000000..fd9c31e6ec --- /dev/null +++ b/oap-server/server-tools/data-generator/src/test/resources/segment.tpl.json @@ -0,0 +1,122 @@ +{ + "traceId": { + "type": "uuid", + "changingFrequency": "1" + }, + "segments": { + "type": "randomList", + "size": 5, + "item": { + "serviceInstanceName": { + "type": "randomString", + "length": "10", + "letters": true, + "numbers": true, + "domainSize": 10 + }, + "serviceName": { + "type": "randomString", + "prefix": "service_", + "length": "1", + "letters": false, + "numbers": true, + "domainSize": 10 + }, + "endpointName": { + "type": "randomString", + "length": "10", + "prefix": "test_", + "letters": true, + "numbers": true, + "domainSize": 10 + }, + "error": { + "type": "randomInt", + "min": 1, + "max": 1 + }, + "tags": { + "type": "randomList", + "size": 5, + "item": { + "key": { + "type": "randomString", + "length": "10", + "prefix": "test_tag_", + "letters": true, + "numbers": true, + "domainSize": 5 + }, + "value": { + "type": "randomString", + "length": "10", + "prefix": "test_value_", + "letters": true, + "numbers": true, + "domainSize": 10 + } + } + }, + "spans": { + "type": "randomList", + "size": 5, + "item": { + "latency": { + "type": "randomInt", + "min": 100, + "max": 1000 + }, + "operationName": { + "type": "randomString", + "length": "10", + "prefix": "test_endpoint_", + "letters": true, + "numbers": true + }, + "peer": { + "type": "randomString", + "length": "10", + "prefix": "test_peer_", + "letters": true, + "numbers": true + }, + "spanLayer": { + "type": "randomInt", + "min": "0", + "max": "4" + }, + "componentId": { + "type": "randomInt", + "min": "0", + "max": "4" + }, + "error": { + "type": "randomBool", + "possibility": "0.2" + }, + "tags": { + "type": "randomList", + "size": 5, + "item": { + "key": { + "type": "randomString", + "length": "10", + "prefix": "test_tag_key_", + "letters": true, + "numbers": true, + "domainSize": 10 + }, + "value": { + "type": "randomString", + "length": "10", + "prefix": "test_tag_val_", + "letters": true, + "numbers": true + } + } + } + } + } + } + } +} \ No newline at end of file