This is an automated email from the ASF dual-hosted git repository. hanahmily pushed a commit to branch trace/sidx in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 5132ecdefe7cd861f2ff68fbe7d8614f5ab8ac9f Author: Gao Hongtao <[email protected]> AuthorDate: Sun Aug 31 18:17:52 2025 +0800 Enhance trace data handling by introducing base64 encoding for span fields during YAML unmarshaling and decoding during JSON marshaling. Updated test cases to reflect new ordering options for trace queries. --- banyand/internal/sidx/block.go | 1 + banyand/query/processor.go | 7 +- test/cases/trace/data/data.go | 83 +++++++++++++++++++++- .../data/input/eq_endpoint_order_duration_asc.yml | 29 ++++++++ .../input/eq_service_instance_order_time_asc.yml | 29 ++++++++ .../data/input/eq_service_order_timestamp_desc.yml | 29 ++++++++ .../data/want/eq_endpoint_order_duration_asc.yml | 20 ++++++ .../want/eq_service_instance_order_time_asc.yml | 21 ++++++ .../data/want/eq_service_order_timestamp_desc.yml | 23 ++++++ test/cases/trace/trace.go | 3 +- 10 files changed, 240 insertions(+), 5 deletions(-) diff --git a/banyand/internal/sidx/block.go b/banyand/internal/sidx/block.go index 77bc1c4c..5dfa535f 100644 --- a/banyand/internal/sidx/block.go +++ b/banyand/internal/sidx/block.go @@ -87,6 +87,7 @@ func (b *block) reset() { // Clear tag map but keep the map itself for k := range b.tags { + releaseTagData(b.tags[k]) delete(b.tags, k) } } diff --git a/banyand/query/processor.go b/banyand/query/processor.go index b711ec20..b9076b3c 100644 --- a/banyand/query/processor.go +++ b/banyand/query/processor.go @@ -534,6 +534,9 @@ func (p *traceQueryProcessor) executeQuery(ctx context.Context, queryCriteria *t // Convert model.TraceResult iterator to tracev1.QueryResponse format var spans []*tracev1.Span + // Check if trace ID tag should be included based on tag projection + shouldIncludeTraceID := slices.Contains(queryCriteria.TagProjection, traceIDTagName) + for { result, hasNext := resultIterator.Next() if !hasNext { @@ -555,8 +558,8 @@ func (p *traceQueryProcessor) executeQuery(ctx context.Context, queryCriteria *t } } - // Add trace ID tag to each span - if traceIDTagName != "" && result.TID != "" { + // Add trace ID tag to each span if it should be included + if shouldIncludeTraceID && result.TID != "" { traceTags = append(traceTags, &modelv1.Tag{ Key: traceIDTagName, Value: &modelv1.TagValue{ diff --git a/test/cases/trace/data/data.go b/test/cases/trace/data/data.go index 90de5657..1ad03a42 100644 --- a/test/cases/trace/data/data.go +++ b/test/cases/trace/data/data.go @@ -21,6 +21,7 @@ package data import ( "context" "embed" + "encoding/base64" "encoding/json" "fmt" "io" @@ -82,7 +83,8 @@ var VerifyFn = func(innerGm gm.Gomega, sharedContext helpers.SharedContext, args ww, err := wantFS.ReadFile("want/" + args.Want + ".yml") innerGm.Expect(err).NotTo(gm.HaveOccurred()) want := &tracev1.QueryResponse{} - helpers.UnmarshalYAML(ww, want) + unmarshalYAMLWithSpanEncoding(ww, want) + if args.DisOrder { slices.SortFunc(want.Spans, func(a, b *tracev1.Span) int { // Sort by first tag value for consistency @@ -105,7 +107,7 @@ var VerifyFn = func(innerGm gm.Gomega, sharedContext helpers.SharedContext, args extra...)). To(gm.BeTrue(), func() string { var j []byte - j, err = protojson.Marshal(resp) + j, err = marshalToJSONWithStringBytes(resp) if err != nil { return err.Error() } @@ -203,3 +205,80 @@ func WriteToGroup(conn *grpclib.ClientConn, name, group, fileName string, baseTi return err }, flags.EventuallyTimeout).Should(gm.Equal(io.EOF)) } + +// unmarshalYAMLWithSpanEncoding decodes YAML with special handling for span data. +// It converts plain strings in the YAML to base64 encoded strings before protobuf unmarshaling. +func unmarshalYAMLWithSpanEncoding(yamlData []byte, response *tracev1.QueryResponse) { + // First convert YAML to JSON + j, err := yaml.YAMLToJSON(yamlData) + gm.Expect(err).NotTo(gm.HaveOccurred()) + + // Parse JSON to modify span fields + var jsonData map[string]interface{} + err = json.Unmarshal(j, &jsonData) + gm.Expect(err).NotTo(gm.HaveOccurred()) + + // Convert span strings to base64 + if spans, ok := jsonData["spans"].([]interface{}); ok { + for _, spanInterface := range spans { + if span, ok := spanInterface.(map[string]interface{}); ok { + if spanValue, ok := span["span"].(string); ok { + // Encode the plain string as base64 + span["span"] = base64.StdEncoding.EncodeToString([]byte(spanValue)) + } + } + } + } + + // Convert back to JSON + modifiedJSON, err := json.Marshal(jsonData) + gm.Expect(err).NotTo(gm.HaveOccurred()) + + // Finally unmarshal to protobuf + gm.Expect(protojson.Unmarshal(modifiedJSON, response)).To(gm.Succeed()) +} + +// marshalToJSONWithStringBytes marshals the QueryResponse to JSON with []byte fields as strings instead of base64. +func marshalToJSONWithStringBytes(resp *tracev1.QueryResponse) ([]byte, error) { + // First marshal to JSON using protojson + j, err := protojson.Marshal(resp) + if err != nil { + return nil, err + } + + // Parse the JSON to modify byte fields + var jsonData map[string]interface{} + err = json.Unmarshal(j, &jsonData) + if err != nil { + return nil, err + } + + // Convert base64 encoded span fields back to strings + if spans, ok := jsonData["spans"].([]interface{}); ok { + for _, spanInterface := range spans { + if span, ok := spanInterface.(map[string]interface{}); ok { + if spanB64, ok := span["span"].(string); ok { + // Decode base64 back to original string + spanBytes, err := base64.StdEncoding.DecodeString(spanB64) + if err != nil { + // If it's not valid base64, keep the original value + continue + } + span["span"] = string(spanBytes) + } + } + } + } + + // Convert back to JSON + return json.Marshal(jsonData) +} + +// GetSpanDataAsString extracts the span data as a string from a Span. +// This converts the raw bytes back to the original string value like "span5_trace_data". +func GetSpanDataAsString(span *tracev1.Span) string { + if span == nil || len(span.Span) == 0 { + return "" + } + return string(span.Span) +} diff --git a/test/cases/trace/data/input/eq_endpoint_order_duration_asc.yml b/test/cases/trace/data/input/eq_endpoint_order_duration_asc.yml new file mode 100644 index 00000000..0458d6d0 --- /dev/null +++ b/test/cases/trace/data/input/eq_endpoint_order_duration_asc.yml @@ -0,0 +1,29 @@ +# Licensed to Apache Software Foundation (ASF) under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Apache Software Foundation (ASF) licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: "sw" +groups: ["test-trace-group"] +criteria: + condition: + name: "endpoint_id" + op: "BINARY_OP_EQ" + value: + str: + value: "/home_endpoint" +order_by: + index_rule_name: "duration" + sort: "SORT_ASC" diff --git a/test/cases/trace/data/input/eq_service_instance_order_time_asc.yml b/test/cases/trace/data/input/eq_service_instance_order_time_asc.yml new file mode 100644 index 00000000..3d31c3b9 --- /dev/null +++ b/test/cases/trace/data/input/eq_service_instance_order_time_asc.yml @@ -0,0 +1,29 @@ +# Licensed to Apache Software Foundation (ASF) under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Apache Software Foundation (ASF) licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: "sw" +groups: ["test-trace-group"] +criteria: + condition: + name: "service_instance_id" + op: "BINARY_OP_EQ" + value: + str: + value: "webapp_instance_1" +order_by: + index_rule_name: "timestamp" + sort: "SORT_ASC" diff --git a/test/cases/trace/data/input/eq_service_order_timestamp_desc.yml b/test/cases/trace/data/input/eq_service_order_timestamp_desc.yml new file mode 100644 index 00000000..e3be5706 --- /dev/null +++ b/test/cases/trace/data/input/eq_service_order_timestamp_desc.yml @@ -0,0 +1,29 @@ +# Licensed to Apache Software Foundation (ASF) under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Apache Software Foundation (ASF) licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: "sw" +groups: ["test-trace-group"] +criteria: + condition: + name: "service_id" + op: "BINARY_OP_EQ" + value: + str: + value: "webapp_service" +order_by: + index_rule_name: "timestamp" + sort: "SORT_DESC" diff --git a/test/cases/trace/data/want/eq_endpoint_order_duration_asc.yml b/test/cases/trace/data/want/eq_endpoint_order_duration_asc.yml new file mode 100644 index 00000000..457a547b --- /dev/null +++ b/test/cases/trace/data/want/eq_endpoint_order_duration_asc.yml @@ -0,0 +1,20 @@ +# Licensed to Apache Software Foundation (ASF) under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Apache Software Foundation (ASF) licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +spans: + - span: span3_trace_data + - span: span1_trace_data diff --git a/test/cases/trace/data/want/eq_service_instance_order_time_asc.yml b/test/cases/trace/data/want/eq_service_instance_order_time_asc.yml new file mode 100644 index 00000000..725839c7 --- /dev/null +++ b/test/cases/trace/data/want/eq_service_instance_order_time_asc.yml @@ -0,0 +1,21 @@ +# Licensed to Apache Software Foundation (ASF) under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Apache Software Foundation (ASF) licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +spans: + - span: span1_trace_data + - span: span3_trace_data + - span: span5_trace_data diff --git a/test/cases/trace/data/want/eq_service_order_timestamp_desc.yml b/test/cases/trace/data/want/eq_service_order_timestamp_desc.yml new file mode 100644 index 00000000..2c9aa31e --- /dev/null +++ b/test/cases/trace/data/want/eq_service_order_timestamp_desc.yml @@ -0,0 +1,23 @@ +# Licensed to Apache Software Foundation (ASF) under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Apache Software Foundation (ASF) licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +spans: + - span: span5_trace_data + - span: span4_trace_data + - span: span3_trace_data + - span: span2_trace_data + - span: span1_trace_data diff --git a/test/cases/trace/trace.go b/test/cases/trace/trace.go index 4ade3006..9e1e285c 100644 --- a/test/cases/trace/trace.go +++ b/test/cases/trace/trace.go @@ -43,5 +43,6 @@ var _ = g.DescribeTable("Scanning Traces", func(args helpers.Args) { }, flags.EventuallyTimeout).Should(gm.Succeed()) }, g.Entry("query by trace id", helpers.Args{Input: "eq_trace_id", Duration: 1 * time.Hour}), - g.FEntry("order by timestamp", helpers.Args{Input: "order_timestamp_desc", Duration: 1 * time.Hour}), + g.Entry("order by timestamp", helpers.Args{Input: "order_timestamp_desc", Duration: 1 * time.Hour}), + g.FEntry("order by duration", helpers.Args{Input: "order_duration_desc", Duration: 1 * time.Hour}), )
