This is an automated email from the ASF dual-hosted git repository. wusheng pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push: new eeef140032 Fix several issues of the data generator and BanyanDB plugin (#12047) eeef140032 is described below commit eeef140032f16d3fae78f61ad7bd03e2ffed48f7 Author: Gao Hongtao <hanahm...@gmail.com> AuthorDate: Fri Mar 22 09:10:17 2024 +0800 Fix several issues of the data generator and BanyanDB plugin (#12047) --- docs/en/setup/backend/segment-template.json | 38 +++++++--------------- .../storage/plugin/banyandb/BanyanDBConverter.java | 35 ++++++++++++++++++-- .../skywalking/restapi/SegmentGenerator.java | 17 +++++----- .../restapi/SegmentGeneratorHandler.java | 15 ++++++++- .../apache/skywalking/restapi/SegmentRequest.java | 32 ++++++++++++++++-- .../apache/skywalking/restapi/SpanGenerator.java | 17 +++++----- .../skywalking/restapi/SegmentGeneratorTest.java | 24 ++++++++++++-- .../src/test/resources/segment.tpl.json | 38 +++++++--------------- 8 files changed, 138 insertions(+), 78 deletions(-) diff --git a/docs/en/setup/backend/segment-template.json b/docs/en/setup/backend/segment-template.json index 0cfc9c1a39..c07a98f382 100644 --- a/docs/en/setup/backend/segment-template.json +++ b/docs/en/setup/backend/segment-template.json @@ -3,25 +3,21 @@ "type": "uuid", "changingFrequency": "1" }, + "serviceInstanceName": { + "type": "randomString", + "length": "10", + "letters": true, + "numbers": true, + "domainSize": 10 + }, + "serviceName": { + "type": "fixedString", + "value": "service_" + }, "segments": { "type": "randomList", "size": 5, "item": { - "serviceInstanceName": { - "type": "randomString", - "length": "10", - "letters": true, - "numbers": true, - "domainSize": 10 - }, - "serviceName": { - "type": "randomString", - "prefix": "service_", - "length": "1", - "letters": false, - "numbers": true, - "domainSize": 10 - }, "endpointName": { "type": "randomString", "length": "10", @@ -73,18 +69,6 @@ "letters": true, "numbers": true }, - "peer": { - "type": "randomString", - "length": "10", - "prefix": "test_peer_", - "letters": true, - "numbers": true - }, - "spanLayer": { - "type": "randomInt", - "min": "0", - "max": "4" - }, "componentId": { "type": "randomInt", "min": "0", diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBConverter.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBConverter.java index 270fca5135..cbd8ecd120 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBConverter.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBConverter.java @@ -29,6 +29,7 @@ import org.apache.skywalking.banyandb.v1.client.StreamWrite; import org.apache.skywalking.banyandb.v1.client.TagAndValue; import org.apache.skywalking.banyandb.v1.client.grpc.exception.BanyanDBException; import org.apache.skywalking.banyandb.v1.client.metadata.Serializable; +import org.apache.skywalking.oap.server.core.analysis.Layer; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; import org.apache.skywalking.oap.server.core.analysis.record.Record; @@ -210,8 +211,8 @@ public class BanyanDBConverter { return TagAndValue.binaryTagValue(ByteUtil.double2Bytes((double) value)); } else if (StorageDataComplexObject.class.isAssignableFrom(clazz)) { return TagAndValue.stringTagValue(((StorageDataComplexObject<?>) value).toStorageData()); - } else if (clazz.isEnum()) { - return TagAndValue.longTagValue((int) value); + } else if (Layer.class.equals(clazz)) { + return TagAndValue.longTagValue(((Integer) value).longValue()); } else if (JsonObject.class.equals(clazz)) { return TagAndValue.stringTagValue((String) value); } else if (byte[].class.equals(clazz)) { @@ -250,15 +251,24 @@ public class BanyanDBConverter { return TimeBucket.getTimeBucket(dataPoint.getTimestamp(), schema.getMetadata().getDownSampling()); } MetadataRegistry.ColumnSpec spec = schema.getSpec(fieldName); + Class<?> clazz = spec.getColumnClass(); switch (spec.getColumnType()) { case TAG: - if (double.class.equals(spec.getColumnClass())) { + Object tv = dataPoint.getTagValue(fieldName); + if (tv == null) { + return defaultValue(clazz); + } + if (double.class.equals(clazz)) { return ByteUtil.bytes2Double(dataPoint.getTagValue(fieldName)); } else { return dataPoint.getTagValue(fieldName); } case FIELD: default: + Object fv = dataPoint.getFieldValue(fieldName); + if (fv == null) { + return defaultValue(clazz); + } if (double.class.equals(spec.getColumnClass())) { return ByteUtil.bytes2Double(dataPoint.getFieldValue(fieldName)); } else { @@ -272,5 +282,24 @@ public class BanyanDBConverter { // TODO: double may be a field? return dataPoint.getFieldValue(fieldName); } + + private Object defaultValue(Class<?> clazz) { + if (int.class.equals(clazz) || Integer.class.equals(clazz)) { + return 0; + } else if (Long.class.equals(clazz) || long.class.equals(clazz)) { + return 0L; + } else if (String.class.equals(clazz)) { + return ""; + } else if (Double.class.equals(clazz) || double.class.equals(clazz)) { + return 0D; + } else if (StorageDataComplexObject.class.isAssignableFrom(clazz)) { + return ""; + } else if (JsonObject.class.equals(clazz)) { + return ""; + } else if (byte[].class.equals(clazz)) { + return new byte[]{}; + } + throw new IllegalStateException(clazz.getSimpleName() + " is not supported"); + } } } diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGenerator.java index 899a3c164e..8ac2b195b0 100644 --- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGenerator.java +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGenerator.java @@ -45,8 +45,6 @@ import java.util.stream.IntStream; public class SegmentGenerator implements Generator<SegmentGenerator.SegmentContext, SegmentGenerator.SegmentResult> { private Generator<String, String> segmentId; - private Generator<String, String> serviceName; - private Generator<String, String> serviceInstanceName; private Generator<String, String> endpointName; private Generator<Object, Long> error; private Generator<Object, List<TagGenerator>> tags; @@ -55,8 +53,8 @@ public class SegmentGenerator implements Generator<SegmentGenerator.SegmentConte @Override public SegmentResult next(SegmentContext ctx) { long now = System.currentTimeMillis(); - final String serviceName = getServiceName().next(null); - final String serviceInstanceName = getServiceInstanceName().next(serviceName); + final String serviceName = ctx.serviceName; + final String serviceInstanceName = ctx.serviceInstanceName; final String endpointName = getEndpointName().next(null); if (segmentId == null) { StringGenerator.Builder segmentIdBuilder = new StringGenerator.Builder(); @@ -75,7 +73,8 @@ public class SegmentGenerator implements Generator<SegmentGenerator.SegmentConte .setParentService(parentSegment.segmentObject.getService()) .setParentSpanId(span.getSpanId()) .setParentTraceSegmentId(parentSegment.segment.getSegmentId()) - .setNetworkAddressUsedAtPeer(span.getPeer()) + .setParentEndpoint(IDManager.EndpointID.analysisId(parentSegment.segment.getEndpointId()).getEndpointName()) + .setNetworkAddressUsedAtPeer(serviceInstanceName) .build())) .orElse(null); final String segmentId = getSegmentId().next(null); @@ -90,7 +89,7 @@ public class SegmentGenerator implements Generator<SegmentGenerator.SegmentConte IntStream.range(0, size) .mapToObj(i -> { SpanGenerator sg = spanGenerators.get(i); - return sg.next(new SpanGenerator.SpanGeneratorContext(i, size, sr)); + return sg.next(new SpanGenerator.SpanGeneratorContext(i, size, sr, ctx.peer)); }) .collect(Collectors.<SpanObject>toList())) .setService(serviceName) @@ -126,14 +125,16 @@ public class SegmentGenerator implements Generator<SegmentGenerator.SegmentConte .map(tg -> tg.next(null)) .collect(Collectors.<Tag>toList())); - segment.setDataBinary(segmentObj.toByteArray()); return new SegmentResult(segment, segmentObj); } @RequiredArgsConstructor public static class SegmentContext { final String traceId; - final SegmentResult parentSegment; + final String serviceName; + final String serviceInstanceName; + String peer; + SegmentResult parentSegment; } @RequiredArgsConstructor diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGeneratorHandler.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGeneratorHandler.java index 0b018324c3..84a5392021 100644 --- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGeneratorHandler.java +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGeneratorHandler.java @@ -61,6 +61,8 @@ public class SegmentGeneratorHandler { public HttpResponse generateMockSegments( @Default("0") @Param("size") int size, @Default("0") @Param("qps") int qps, + @Default("0") @Param("duration") int duration, + @Default("") @Param("group") String group, @RequestObject SegmentRequest request) { if (size > 0 && qps > 0) { @@ -71,8 +73,9 @@ public class SegmentGeneratorHandler { } log.info("Generate {} mock segments, qps: {}, template: {}", size, qps, request); + request.init(group); final IntConsumer generator = unused -> { - final List<SegmentGenerator.SegmentResult> segments = request.next(null); + final List<SegmentGenerator.SegmentResult> segments = request.next(group); log.debug("Generating segment: {}", (Object) segments); segments.forEach(s -> { segmentParserService.send(s.segmentObject); @@ -102,6 +105,16 @@ public class SegmentGeneratorHandler { log.error("Exception in future: ", f.cause()); } }); + final int durationSeconds = duration > 0 ? duration : Integer.MAX_VALUE; + future.addListener(f -> { + try { + Thread.sleep(durationSeconds * 1000L); + future.cancel(true); + log.info("Generate mock segments is canceled: requestId: {}", requestId); + } catch (InterruptedException e) { + log.error("Interrupted", e); + } + }); return HttpResponse.of(MediaType.PLAIN_TEXT, requestId); } diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentRequest.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentRequest.java index 8fa57d30ef..7e45cf1fd2 100644 --- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentRequest.java +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentRequest.java @@ -20,28 +20,56 @@ package org.apache.skywalking.restapi; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.common.base.Strings; import lombok.Data; import org.apache.skywalking.generator.Generator; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; @Data @JsonTypeInfo(use = JsonTypeInfo.Id.NONE) public final class SegmentRequest implements Generator<Object, List<SegmentGenerator.SegmentResult>> { private Generator<String, String> traceId; + private Generator<String, String> serviceName; + private Generator<String, String> serviceInstanceName; private Generator<Object, List<SegmentGenerator>> segments; + private List<String> serviceList; + + void init(String group) { + String prefix = getServiceName().next(""); + final List<SegmentGenerator> segments = getSegments().next(""); + serviceList = IntStream.range(0, segments.size()).mapToObj(i -> { + if (Strings.isNullOrEmpty(group)) { + return prefix + i; + } + return group + "::" + prefix + i; + }).collect(Collectors.toList()); + } + @Override public List<SegmentGenerator.SegmentResult> next(Object ignored) { final String traceId = getTraceId().next(null); final List<SegmentGenerator> segments = getSegments().next(traceId); SegmentGenerator.SegmentResult last = null; + List<SegmentGenerator.SegmentContext> context = IntStream.range(0, segments.size()).mapToObj(i -> + new SegmentGenerator.SegmentContext(traceId, serviceList.get(i), getServiceInstanceName().next(serviceList.get(i))) + ).collect(Collectors.toList()); List<SegmentGenerator.SegmentResult> result = new ArrayList<>(segments.size()); - for (SegmentGenerator each : segments) { - last = each.next(new SegmentGenerator.SegmentContext(traceId, last)); + for (int i = 0; i < segments.size(); i++) { + SegmentGenerator.SegmentContext ctx = context.get(i); + ctx.parentSegment = last; + if (i < segments.size() - 1) { + ctx.peer = context.get(i + 1).serviceInstanceName; + } + last = segments.get(i).next(ctx); result.add(last); } + return result; } + } diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SpanGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SpanGenerator.java index 708e58fe0b..2e1920046a 100644 --- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SpanGenerator.java +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SpanGenerator.java @@ -39,8 +39,6 @@ public final class SpanGenerator implements Generator<SpanGenerator.SpanGenerato private Generator<Object, Long> endTime; private Generator<Object, Long> latency; private Generator<String, String> operationName; - private Generator<String, String> peer; - private Generator<Object, Long> spanLayer; private Generator<Object, Long> componentId; private Generator<Object, Boolean> error; private Generator<Object, List<TagGenerator>> tags; @@ -71,16 +69,20 @@ public final class SpanGenerator implements Generator<SpanGenerator.SpanGenerato .setValue(it.getValue()).build()) .collect(Collectors.toList())); if (ctx.index == 0) { - sob.setSpanLayer(SpanLayer.forNumber(getSpanLayer().next(null).intValue())) + sob.setSpanLayer(SpanLayer.Http) .setSpanType(SpanType.Entry); if (ctx.ref != null) { sob.addRefs(ctx.ref); } } else if (ctx.length > 1 && ctx.index == ctx.length - 1) { - sob.setSpanType(SpanType.Exit).setPeer(getPeer().next(null)).setSpanLayer(SpanLayer.Database); + sob.setSpanType(SpanType.Exit); + if (ctx.peer != null) { + sob.setPeer(ctx.peer).setSpanLayer(SpanLayer.Http); + } else { + sob.setPeer("peer-database").setSpanLayer(SpanLayer.Database); + } } else { - sob.setSpanLayer(SpanLayer.forNumber(getSpanLayer().next(null).intValue())) - .setSpanType(SpanType.Local); + sob.setSpanType(SpanType.Local); } return sob.build(); } @@ -88,8 +90,6 @@ public final class SpanGenerator implements Generator<SpanGenerator.SpanGenerato @Override public void reset() { operationName.reset(); - peer.reset(); - spanLayer.reset(); componentId.reset(); error.reset(); tags.reset(); @@ -100,5 +100,6 @@ public final class SpanGenerator implements Generator<SpanGenerator.SpanGenerato final int index; final int length; final SegmentReference ref; + final String peer; } } diff --git a/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/restapi/SegmentGeneratorTest.java b/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/restapi/SegmentGeneratorTest.java index 29c9228fdb..788b9f2e7f 100644 --- a/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/restapi/SegmentGeneratorTest.java +++ b/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/restapi/SegmentGeneratorTest.java @@ -26,9 +26,12 @@ import java.io.File; import java.io.IOException; import java.net.URISyntaxException; import java.net.URL; +import java.util.HashSet; import java.util.List; +import java.util.Set; import static graphql.Assert.assertFalse; +import static graphql.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertNotNull; class SegmentGeneratorTest { @@ -40,7 +43,24 @@ class SegmentGeneratorTest { assertNotNull(url); File jsonFile = new File(url.toURI()); SegmentRequest sr = objectMapper.readValue(jsonFile, SegmentRequest.class); - List<SegmentGenerator.SegmentResult> ss = sr.next(null); - assertFalse(ss.isEmpty()); + sr.init(""); + Set<String> serviceSet = new HashSet<>(); + Set<String> serviceInstanceSet = new HashSet<>(); + Set<String> endpointSet = new HashSet<>(); + for (int i = 0; i < 1000; i++) { + List<SegmentGenerator.SegmentResult> ss = sr.next(null); + assertFalse(ss.isEmpty()); + for (SegmentGenerator.SegmentResult s : ss) { + serviceSet.add(s.segmentObject.getService()); + serviceInstanceSet.add(s.segmentObject.getServiceInstance()); + endpointSet.add(s.segment.getEndpointId()); + } + } + assertTrue(serviceSet.size() > 1); + assertTrue(serviceSet.size() <= 10); + assertTrue(serviceInstanceSet.size() > 1); + assertTrue(serviceInstanceSet.size() <= 100); + assertTrue(endpointSet.size() > 1); + assertTrue(endpointSet.size() <= 100); } } \ No newline at end of file diff --git a/oap-server/server-tools/data-generator/src/test/resources/segment.tpl.json b/oap-server/server-tools/data-generator/src/test/resources/segment.tpl.json index fd9c31e6ec..3597234abf 100644 --- a/oap-server/server-tools/data-generator/src/test/resources/segment.tpl.json +++ b/oap-server/server-tools/data-generator/src/test/resources/segment.tpl.json @@ -3,25 +3,21 @@ "type": "uuid", "changingFrequency": "1" }, + "serviceInstanceName": { + "type": "randomString", + "length": "10", + "letters": true, + "numbers": true, + "domainSize": 10 + }, + "serviceName": { + "type": "fixedString", + "value": "service_" + }, "segments": { "type": "randomList", "size": 5, "item": { - "serviceInstanceName": { - "type": "randomString", - "length": "10", - "letters": true, - "numbers": true, - "domainSize": 10 - }, - "serviceName": { - "type": "randomString", - "prefix": "service_", - "length": "1", - "letters": false, - "numbers": true, - "domainSize": 10 - }, "endpointName": { "type": "randomString", "length": "10", @@ -73,18 +69,6 @@ "letters": true, "numbers": true }, - "peer": { - "type": "randomString", - "length": "10", - "prefix": "test_peer_", - "letters": true, - "numbers": true - }, - "spanLayer": { - "type": "randomInt", - "min": "0", - "max": "4" - }, "componentId": { "type": "randomInt", "min": "0",