This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new eeef140032 Fix several issues of the data generator and BanyanDB
plugin (#12047)
eeef140032 is described below
commit eeef140032f16d3fae78f61ad7bd03e2ffed48f7
Author: Gao Hongtao <hanahm...@gmail.com>
AuthorDate: Fri Mar 22 09:10:17 2024 +0800
Fix several issues of the data generator and BanyanDB plugin (#12047)
---
docs/en/setup/backend/segment-template.json | 38 +++++++---------------
.../storage/plugin/banyandb/BanyanDBConverter.java | 35 ++++++++++++++++++--
.../skywalking/restapi/SegmentGenerator.java | 17 +++++-----
.../restapi/SegmentGeneratorHandler.java | 15 ++++++++-
.../apache/skywalking/restapi/SegmentRequest.java | 32 ++++++++++++++++--
.../apache/skywalking/restapi/SpanGenerator.java | 17 +++++-----
.../skywalking/restapi/SegmentGeneratorTest.java | 24 ++++++++++++--
.../src/test/resources/segment.tpl.json | 38 +++++++---------------
8 files changed, 138 insertions(+), 78 deletions(-)
diff --git a/docs/en/setup/backend/segment-template.json
b/docs/en/setup/backend/segment-template.json
index 0cfc9c1a39..c07a98f382 100644
--- a/docs/en/setup/backend/segment-template.json
+++ b/docs/en/setup/backend/segment-template.json
@@ -3,25 +3,21 @@
"type": "uuid",
"changingFrequency": "1"
},
+ "serviceInstanceName": {
+ "type": "randomString",
+ "length": "10",
+ "letters": true,
+ "numbers": true,
+ "domainSize": 10
+ },
+ "serviceName": {
+ "type": "fixedString",
+ "value": "service_"
+ },
"segments": {
"type": "randomList",
"size": 5,
"item": {
- "serviceInstanceName": {
- "type": "randomString",
- "length": "10",
- "letters": true,
- "numbers": true,
- "domainSize": 10
- },
- "serviceName": {
- "type": "randomString",
- "prefix": "service_",
- "length": "1",
- "letters": false,
- "numbers": true,
- "domainSize": 10
- },
"endpointName": {
"type": "randomString",
"length": "10",
@@ -73,18 +69,6 @@
"letters": true,
"numbers": true
},
- "peer": {
- "type": "randomString",
- "length": "10",
- "prefix": "test_peer_",
- "letters": true,
- "numbers": true
- },
- "spanLayer": {
- "type": "randomInt",
- "min": "0",
- "max": "4"
- },
"componentId": {
"type": "randomInt",
"min": "0",
diff --git
a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBConverter.java
b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBConverter.java
index 270fca5135..cbd8ecd120 100644
---
a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBConverter.java
+++
b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBConverter.java
@@ -29,6 +29,7 @@ import org.apache.skywalking.banyandb.v1.client.StreamWrite;
import org.apache.skywalking.banyandb.v1.client.TagAndValue;
import
org.apache.skywalking.banyandb.v1.client.grpc.exception.BanyanDBException;
import org.apache.skywalking.banyandb.v1.client.metadata.Serializable;
+import org.apache.skywalking.oap.server.core.analysis.Layer;
import org.apache.skywalking.oap.server.core.analysis.TimeBucket;
import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.analysis.record.Record;
@@ -210,8 +211,8 @@ public class BanyanDBConverter {
return TagAndValue.binaryTagValue(ByteUtil.double2Bytes((double)
value));
} else if (StorageDataComplexObject.class.isAssignableFrom(clazz)) {
return TagAndValue.stringTagValue(((StorageDataComplexObject<?>)
value).toStorageData());
- } else if (clazz.isEnum()) {
- return TagAndValue.longTagValue((int) value);
+ } else if (Layer.class.equals(clazz)) {
+ return TagAndValue.longTagValue(((Integer) value).longValue());
} else if (JsonObject.class.equals(clazz)) {
return TagAndValue.stringTagValue((String) value);
} else if (byte[].class.equals(clazz)) {
@@ -250,15 +251,24 @@ public class BanyanDBConverter {
return TimeBucket.getTimeBucket(dataPoint.getTimestamp(),
schema.getMetadata().getDownSampling());
}
MetadataRegistry.ColumnSpec spec = schema.getSpec(fieldName);
+ Class<?> clazz = spec.getColumnClass();
switch (spec.getColumnType()) {
case TAG:
- if (double.class.equals(spec.getColumnClass())) {
+ Object tv = dataPoint.getTagValue(fieldName);
+ if (tv == null) {
+ return defaultValue(clazz);
+ }
+ if (double.class.equals(clazz)) {
return
ByteUtil.bytes2Double(dataPoint.getTagValue(fieldName));
} else {
return dataPoint.getTagValue(fieldName);
}
case FIELD:
default:
+ Object fv = dataPoint.getFieldValue(fieldName);
+ if (fv == null) {
+ return defaultValue(clazz);
+ }
if (double.class.equals(spec.getColumnClass())) {
return
ByteUtil.bytes2Double(dataPoint.getFieldValue(fieldName));
} else {
@@ -272,5 +282,24 @@ public class BanyanDBConverter {
// TODO: double may be a field?
return dataPoint.getFieldValue(fieldName);
}
+
+ private Object defaultValue(Class<?> clazz) {
+ if (int.class.equals(clazz) || Integer.class.equals(clazz)) {
+ return 0;
+ } else if (Long.class.equals(clazz) || long.class.equals(clazz)) {
+ return 0L;
+ } else if (String.class.equals(clazz)) {
+ return "";
+ } else if (Double.class.equals(clazz) ||
double.class.equals(clazz)) {
+ return 0D;
+ } else if (StorageDataComplexObject.class.isAssignableFrom(clazz))
{
+ return "";
+ } else if (JsonObject.class.equals(clazz)) {
+ return "";
+ } else if (byte[].class.equals(clazz)) {
+ return new byte[]{};
+ }
+ throw new IllegalStateException(clazz.getSimpleName() + " is not
supported");
+ }
}
}
diff --git
a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGenerator.java
b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGenerator.java
index 899a3c164e..8ac2b195b0 100644
---
a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGenerator.java
+++
b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGenerator.java
@@ -45,8 +45,6 @@ import java.util.stream.IntStream;
public class SegmentGenerator implements
Generator<SegmentGenerator.SegmentContext, SegmentGenerator.SegmentResult> {
private Generator<String, String> segmentId;
- private Generator<String, String> serviceName;
- private Generator<String, String> serviceInstanceName;
private Generator<String, String> endpointName;
private Generator<Object, Long> error;
private Generator<Object, List<TagGenerator>> tags;
@@ -55,8 +53,8 @@ public class SegmentGenerator implements
Generator<SegmentGenerator.SegmentConte
@Override
public SegmentResult next(SegmentContext ctx) {
long now = System.currentTimeMillis();
- final String serviceName = getServiceName().next(null);
- final String serviceInstanceName =
getServiceInstanceName().next(serviceName);
+ final String serviceName = ctx.serviceName;
+ final String serviceInstanceName = ctx.serviceInstanceName;
final String endpointName = getEndpointName().next(null);
if (segmentId == null) {
StringGenerator.Builder segmentIdBuilder = new
StringGenerator.Builder();
@@ -75,7 +73,8 @@ public class SegmentGenerator implements
Generator<SegmentGenerator.SegmentConte
.setParentService(parentSegment.segmentObject.getService())
.setParentSpanId(span.getSpanId())
.setParentTraceSegmentId(parentSegment.segment.getSegmentId())
- .setNetworkAddressUsedAtPeer(span.getPeer())
+
.setParentEndpoint(IDManager.EndpointID.analysisId(parentSegment.segment.getEndpointId()).getEndpointName())
+
.setNetworkAddressUsedAtPeer(serviceInstanceName)
.build()))
.orElse(null);
final String segmentId = getSegmentId().next(null);
@@ -90,7 +89,7 @@ public class SegmentGenerator implements
Generator<SegmentGenerator.SegmentConte
IntStream.range(0, size)
.mapToObj(i -> {
SpanGenerator sg = spanGenerators.get(i);
- return sg.next(new
SpanGenerator.SpanGeneratorContext(i, size, sr));
+ return sg.next(new
SpanGenerator.SpanGeneratorContext(i, size, sr, ctx.peer));
})
.collect(Collectors.<SpanObject>toList()))
.setService(serviceName)
@@ -126,14 +125,16 @@ public class SegmentGenerator implements
Generator<SegmentGenerator.SegmentConte
.map(tg -> tg.next(null))
.collect(Collectors.<Tag>toList()));
- segment.setDataBinary(segmentObj.toByteArray());
return new SegmentResult(segment, segmentObj);
}
@RequiredArgsConstructor
public static class SegmentContext {
final String traceId;
- final SegmentResult parentSegment;
+ final String serviceName;
+ final String serviceInstanceName;
+ String peer;
+ SegmentResult parentSegment;
}
@RequiredArgsConstructor
diff --git
a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGeneratorHandler.java
b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGeneratorHandler.java
index 0b018324c3..84a5392021 100644
---
a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGeneratorHandler.java
+++
b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGeneratorHandler.java
@@ -61,6 +61,8 @@ public class SegmentGeneratorHandler {
public HttpResponse generateMockSegments(
@Default("0") @Param("size") int size,
@Default("0") @Param("qps") int qps,
+ @Default("0") @Param("duration") int duration,
+ @Default("") @Param("group") String group,
@RequestObject SegmentRequest request) {
if (size > 0 && qps > 0) {
@@ -71,8 +73,9 @@ public class SegmentGeneratorHandler {
}
log.info("Generate {} mock segments, qps: {}, template: {}", size,
qps, request);
+ request.init(group);
final IntConsumer generator = unused -> {
- final List<SegmentGenerator.SegmentResult> segments =
request.next(null);
+ final List<SegmentGenerator.SegmentResult> segments =
request.next(group);
log.debug("Generating segment: {}", (Object) segments);
segments.forEach(s -> {
segmentParserService.send(s.segmentObject);
@@ -102,6 +105,16 @@ public class SegmentGeneratorHandler {
log.error("Exception in future: ", f.cause());
}
});
+ final int durationSeconds = duration > 0 ? duration :
Integer.MAX_VALUE;
+ future.addListener(f -> {
+ try {
+ Thread.sleep(durationSeconds * 1000L);
+ future.cancel(true);
+ log.info("Generate mock segments is canceled: requestId: {}",
requestId);
+ } catch (InterruptedException e) {
+ log.error("Interrupted", e);
+ }
+ });
return HttpResponse.of(MediaType.PLAIN_TEXT, requestId);
}
diff --git
a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentRequest.java
b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentRequest.java
index 8fa57d30ef..7e45cf1fd2 100644
---
a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentRequest.java
+++
b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentRequest.java
@@ -20,28 +20,56 @@
package org.apache.skywalking.restapi;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.google.common.base.Strings;
import lombok.Data;
import org.apache.skywalking.generator.Generator;
import java.util.ArrayList;
import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
@Data
@JsonTypeInfo(use = JsonTypeInfo.Id.NONE)
public final class SegmentRequest implements Generator<Object,
List<SegmentGenerator.SegmentResult>> {
private Generator<String, String> traceId;
+ private Generator<String, String> serviceName;
+ private Generator<String, String> serviceInstanceName;
private Generator<Object, List<SegmentGenerator>> segments;
+ private List<String> serviceList;
+
+ void init(String group) {
+ String prefix = getServiceName().next("");
+ final List<SegmentGenerator> segments = getSegments().next("");
+ serviceList = IntStream.range(0, segments.size()).mapToObj(i -> {
+ if (Strings.isNullOrEmpty(group)) {
+ return prefix + i;
+ }
+ return group + "::" + prefix + i;
+ }).collect(Collectors.toList());
+ }
+
@Override
public List<SegmentGenerator.SegmentResult> next(Object ignored) {
final String traceId = getTraceId().next(null);
final List<SegmentGenerator> segments = getSegments().next(traceId);
SegmentGenerator.SegmentResult last = null;
+ List<SegmentGenerator.SegmentContext> context = IntStream.range(0,
segments.size()).mapToObj(i ->
+ new SegmentGenerator.SegmentContext(traceId, serviceList.get(i),
getServiceInstanceName().next(serviceList.get(i)))
+ ).collect(Collectors.toList());
List<SegmentGenerator.SegmentResult> result = new
ArrayList<>(segments.size());
- for (SegmentGenerator each : segments) {
- last = each.next(new SegmentGenerator.SegmentContext(traceId,
last));
+ for (int i = 0; i < segments.size(); i++) {
+ SegmentGenerator.SegmentContext ctx = context.get(i);
+ ctx.parentSegment = last;
+ if (i < segments.size() - 1) {
+ ctx.peer = context.get(i + 1).serviceInstanceName;
+ }
+ last = segments.get(i).next(ctx);
result.add(last);
}
+
return result;
}
+
}
diff --git
a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SpanGenerator.java
b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SpanGenerator.java
index 708e58fe0b..2e1920046a 100644
---
a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SpanGenerator.java
+++
b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SpanGenerator.java
@@ -39,8 +39,6 @@ public final class SpanGenerator implements
Generator<SpanGenerator.SpanGenerato
private Generator<Object, Long> endTime;
private Generator<Object, Long> latency;
private Generator<String, String> operationName;
- private Generator<String, String> peer;
- private Generator<Object, Long> spanLayer;
private Generator<Object, Long> componentId;
private Generator<Object, Boolean> error;
private Generator<Object, List<TagGenerator>> tags;
@@ -71,16 +69,20 @@ public final class SpanGenerator implements
Generator<SpanGenerator.SpanGenerato
.setValue(it.getValue()).build())
.collect(Collectors.toList()));
if (ctx.index == 0) {
-
sob.setSpanLayer(SpanLayer.forNumber(getSpanLayer().next(null).intValue()))
+ sob.setSpanLayer(SpanLayer.Http)
.setSpanType(SpanType.Entry);
if (ctx.ref != null) {
sob.addRefs(ctx.ref);
}
} else if (ctx.length > 1 && ctx.index == ctx.length - 1) {
-
sob.setSpanType(SpanType.Exit).setPeer(getPeer().next(null)).setSpanLayer(SpanLayer.Database);
+ sob.setSpanType(SpanType.Exit);
+ if (ctx.peer != null) {
+ sob.setPeer(ctx.peer).setSpanLayer(SpanLayer.Http);
+ } else {
+ sob.setPeer("peer-database").setSpanLayer(SpanLayer.Database);
+ }
} else {
-
sob.setSpanLayer(SpanLayer.forNumber(getSpanLayer().next(null).intValue()))
- .setSpanType(SpanType.Local);
+ sob.setSpanType(SpanType.Local);
}
return sob.build();
}
@@ -88,8 +90,6 @@ public final class SpanGenerator implements
Generator<SpanGenerator.SpanGenerato
@Override
public void reset() {
operationName.reset();
- peer.reset();
- spanLayer.reset();
componentId.reset();
error.reset();
tags.reset();
@@ -100,5 +100,6 @@ public final class SpanGenerator implements
Generator<SpanGenerator.SpanGenerato
final int index;
final int length;
final SegmentReference ref;
+ final String peer;
}
}
diff --git
a/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/restapi/SegmentGeneratorTest.java
b/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/restapi/SegmentGeneratorTest.java
index 29c9228fdb..788b9f2e7f 100644
---
a/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/restapi/SegmentGeneratorTest.java
+++
b/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/restapi/SegmentGeneratorTest.java
@@ -26,9 +26,12 @@ import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URL;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import static graphql.Assert.assertFalse;
+import static graphql.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertNotNull;
class SegmentGeneratorTest {
@@ -40,7 +43,24 @@ class SegmentGeneratorTest {
assertNotNull(url);
File jsonFile = new File(url.toURI());
SegmentRequest sr = objectMapper.readValue(jsonFile,
SegmentRequest.class);
- List<SegmentGenerator.SegmentResult> ss = sr.next(null);
- assertFalse(ss.isEmpty());
+ sr.init("");
+ Set<String> serviceSet = new HashSet<>();
+ Set<String> serviceInstanceSet = new HashSet<>();
+ Set<String> endpointSet = new HashSet<>();
+ for (int i = 0; i < 1000; i++) {
+ List<SegmentGenerator.SegmentResult> ss = sr.next(null);
+ assertFalse(ss.isEmpty());
+ for (SegmentGenerator.SegmentResult s : ss) {
+ serviceSet.add(s.segmentObject.getService());
+ serviceInstanceSet.add(s.segmentObject.getServiceInstance());
+ endpointSet.add(s.segment.getEndpointId());
+ }
+ }
+ assertTrue(serviceSet.size() > 1);
+ assertTrue(serviceSet.size() <= 10);
+ assertTrue(serviceInstanceSet.size() > 1);
+ assertTrue(serviceInstanceSet.size() <= 100);
+ assertTrue(endpointSet.size() > 1);
+ assertTrue(endpointSet.size() <= 100);
}
}
\ No newline at end of file
diff --git
a/oap-server/server-tools/data-generator/src/test/resources/segment.tpl.json
b/oap-server/server-tools/data-generator/src/test/resources/segment.tpl.json
index fd9c31e6ec..3597234abf 100644
--- a/oap-server/server-tools/data-generator/src/test/resources/segment.tpl.json
+++ b/oap-server/server-tools/data-generator/src/test/resources/segment.tpl.json
@@ -3,25 +3,21 @@
"type": "uuid",
"changingFrequency": "1"
},
+ "serviceInstanceName": {
+ "type": "randomString",
+ "length": "10",
+ "letters": true,
+ "numbers": true,
+ "domainSize": 10
+ },
+ "serviceName": {
+ "type": "fixedString",
+ "value": "service_"
+ },
"segments": {
"type": "randomList",
"size": 5,
"item": {
- "serviceInstanceName": {
- "type": "randomString",
- "length": "10",
- "letters": true,
- "numbers": true,
- "domainSize": 10
- },
- "serviceName": {
- "type": "randomString",
- "prefix": "service_",
- "length": "1",
- "letters": false,
- "numbers": true,
- "domainSize": 10
- },
"endpointName": {
"type": "randomString",
"length": "10",
@@ -73,18 +69,6 @@
"letters": true,
"numbers": true
},
- "peer": {
- "type": "randomString",
- "length": "10",
- "prefix": "test_peer_",
- "letters": true,
- "numbers": true
- },
- "spanLayer": {
- "type": "randomInt",
- "min": "0",
- "max": "4"
- },
"componentId": {
"type": "randomInt",
"min": "0",
