This is an automated email from the ASF dual-hosted git repository. wusheng pushed a commit to branch groovy-replace in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit 2e7ae6326aaecd1a22e80f46858e5a90485b4056 Author: Wu Sheng <[email protected]> AuthorDate: Sun Mar 1 22:38:15 2026 +0800 Fix all MAL/LAL v2 compiler gaps and add runtime execution comparison MAL compiler fixes (closes 38 previously failing expressions): - Add ternary operator (?:) support in closures (grammar, AST, codegen) - Fix valueEqual() and other primitive-double methods with numeric literal args - Support double-paren argument syntax: sum((['cluster'])) - Handle NUMBER / SampleFamily via MalRuntimeHelper.divReverse() in v2 package - Add variable declarations, map literals, forEach/instance closure types - Add ProcessRegistry class references, improved safe navigation LAL compiler fixes: - Fix null-to-string conversion: use null-safe toStr() instead of String.valueOf() - Add camelToSnake field name fallback for protobuf field access - Add typed execute(FilterSpec, Binding) method signature - Reorganize LAL test scripts into oap-cases/ and feature-cases/ - Add data-driven LALExpressionExecutionTest with 27 test cases MAL checker enhancements: - Add runtime execution comparison (mock SampleFamily data, execute both v1 and v2, compare output samples with labels and values) - Handle increase()/rate() by priming CounterWindow with initial run - Extract tagEqual patterns from expressions for matching mock data All 1,187 MAL + 29 LAL + 22 hierarchy expressions now pass with zero gaps. Co-Authored-By: Claude Opus 4.6 <[email protected]> --- oap-server/analyzer/log-analyzer/pom.xml | 6 + .../log/analyzer/compiler/LALClassGenerator.java | 24 +- .../oap/log/analyzer/compiler/LALScriptModel.java | 2 +- .../oap/log/analyzer/compiler/LALScriptParser.java | 8 + .../skywalking/oap/log/analyzer/dsl/Binding.java | 30 +- .../compiler/LALExpressionExecutionTest.java | 478 +++++++++++++++++++++ .../apache/skywalking/mal/rt/grammar/MALParser.g4 | 32 +- .../meter/analyzer/compiler/MALClassGenerator.java | 306 +++++++++++-- .../analyzer/compiler/MALExpressionModel.java | 102 ++++- .../meter/analyzer/compiler/MALScriptParser.java | 80 +++- .../analyzer/compiler/rt/MalRuntimeHelper.java | 52 +++ .../analyzer/compiler/MALClassGeneratorTest.java | 192 +++++++++ .../analyzer/compiler/MALScriptParserTest.java | 121 ++++++ oap-server/server-starter/pom.xml | 7 + .../oap/server/checker/lal/LalComparisonTest.java | 44 +- .../oap/server/checker/mal/MalComparisonTest.java | 236 +++++++++- .../feature-cases/execution-basic.input.data | 157 +++++++ .../test-lal/feature-cases/execution-basic.yaml | 272 ++++++++++++ .../{default.yaml => oap-cases/default.input.data} | 17 +- .../lal/test-lal/{ => oap-cases}/default.yaml | 0 .../envoy-als.input.data} | 21 +- .../lal/test-lal/{ => oap-cases}/envoy-als.yaml | 0 .../lal/test-lal/oap-cases/k8s-service.input.data | 39 ++ .../lal/test-lal/{ => oap-cases}/k8s-service.yaml | 0 .../lal/test-lal/oap-cases/mesh-dp.input.data | 39 ++ .../lal/test-lal/{ => oap-cases}/mesh-dp.yaml | 0 .../mysql-slowsql.input.data} | 20 +- .../test-lal/{ => oap-cases}/mysql-slowsql.yaml | 0 .../{default.yaml => oap-cases/nginx.input.data} | 18 +- .../lal/test-lal/{ => oap-cases}/nginx.yaml | 0 .../pgsql-slowsql.input.data} | 20 +- .../test-lal/{ => oap-cases}/pgsql-slowsql.yaml | 0 .../redis-slowsql.input.data} | 20 +- .../test-lal/{ => oap-cases}/redis-slowsql.yaml | 0 34 files changed, 2199 insertions(+), 144 deletions(-) diff --git a/oap-server/analyzer/log-analyzer/pom.xml b/oap-server/analyzer/log-analyzer/pom.xml index ff811b1596..180a8435df 100644 --- a/oap-server/analyzer/log-analyzer/pom.xml +++ b/oap-server/analyzer/log-analyzer/pom.xml @@ -55,6 +55,12 @@ <groupId>org.javassist</groupId> <artifactId>javassist</artifactId> </dependency> + <dependency> + <groupId>org.apache.skywalking</groupId> + <artifactId>receiver-proto</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> </dependencies> <build> diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/compiler/LALClassGenerator.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/compiler/LALClassGenerator.java index be5e78d9ef..94895706f6 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/compiler/LALClassGenerator.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/compiler/LALClassGenerator.java @@ -507,8 +507,10 @@ public final class LALClassGenerator { final ConsumerInfo parentInfo, final int[] subCounter) { for (final LALScriptModel.FilterStatement stmt : stmts) { - if (stmt instanceof LALScriptModel.FieldAssignment) { - // SampledTrace fields (processId, latency, etc.) are parsed as FieldAssignment + if (stmt instanceof LALScriptModel.SampledTraceField) { + generateSampledTraceField(sb, + (LALScriptModel.SampledTraceField) stmt); + } else if (stmt instanceof LALScriptModel.FieldAssignment) { generateSampledTraceFieldFromAssignment(sb, (LALScriptModel.FieldAssignment) stmt); } else if (stmt instanceof LALScriptModel.IfBlock) { @@ -767,11 +769,8 @@ public final class LALClassGenerator { private String generateExecuteMethod(final LALScriptModel model, final int[] counter) { final StringBuilder sb = new StringBuilder(); - sb.append("public void execute(Object arg0, Object arg1) {\n"); - sb.append(" ").append(FILTER_SPEC).append(" filterSpec = (") - .append(FILTER_SPEC).append(") arg0;\n"); - sb.append(" ").append(BINDING).append(" binding = (") - .append(BINDING).append(") arg1;\n"); + sb.append("public void execute(").append(FILTER_SPEC) + .append(" filterSpec, ").append(BINDING).append(" binding) {\n"); for (final LALScriptModel.FilterStatement stmt : model.getStatements()) { @@ -861,6 +860,8 @@ public final class LALClassGenerator { + " return ((" + BINDING_PARSED + ") obj).getAt(key);" + " if (obj instanceof java.util.Map)" + " return ((java.util.Map) obj).get(key);" + + " Object protoResult = " + BINDING_PARSED + ".getField(obj, key);" + + " if (protoResult != null) return protoResult;" + " return null;" + "}", ctClass)); @@ -878,6 +879,11 @@ public final class LALClassGenerator { + " return 0;" + "}", ctClass)); + ctClass.addMethod(CtNewMethod.make( + "private static String toStr(Object obj) {" + + " return obj == null ? null : String.valueOf(obj);" + + "}", ctClass)); + ctClass.addMethod(CtNewMethod.make( "private static boolean toBool(Object obj) {" + " if (obj instanceof Boolean) return ((Boolean) obj).booleanValue();" @@ -1044,7 +1050,7 @@ public final class LALClassGenerator { final LALScriptModel.ValueAccess value, final String castType) { if ("String".equals(castType)) { - sb.append("String.valueOf("); + sb.append("toStr("); generateValueAccess(sb, value); sb.append(")"); } else if ("Long".equals(castType)) { @@ -1068,7 +1074,7 @@ public final class LALClassGenerator { final LALScriptModel.ValueAccess value, final String castType) { if ("String".equals(castType)) { - sb.append("String.valueOf("); + sb.append("toStr("); generateValueAccess(sb, value); sb.append(")"); } else { diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/compiler/LALScriptModel.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/compiler/LALScriptModel.java index 8609deb66d..f20a66abdd 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/compiler/LALScriptModel.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/compiler/LALScriptModel.java @@ -197,7 +197,7 @@ public final class LALScriptModel { } @Getter - public static final class SampledTraceField implements SampledTraceStatement { + public static final class SampledTraceField implements SampledTraceStatement, FilterStatement { private final SampledTraceFieldType fieldType; private final ValueAccess value; private final String castType; diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/compiler/LALScriptParser.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/compiler/LALScriptParser.java index 2793ed4dde..e9128221b3 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/compiler/LALScriptParser.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/compiler/LALScriptParser.java @@ -499,6 +499,14 @@ public final class LALScriptParser { stmts.add((FilterStatement) new DropperStatement()); } } + for (final LALParser.SampledTraceStatementContext stc : + ctx.sampledTraceStatement()) { + if (stc.ifStatement() != null) { + stmts.add((FilterStatement) visitIfStatement(stc.ifStatement())); + } else { + stmts.add((FilterStatement) visitSampledTraceField(stc)); + } + } return stmts; } diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/Binding.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/Binding.java index 41404fbdb0..3c60d3df3a 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/Binding.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/Binding.java @@ -206,22 +206,46 @@ public class Binding { return null; } - static Object getField(final Object obj, final String name) { + public static Object getField(final Object obj, final String name) { if (obj instanceof Message) { - final Descriptors.FieldDescriptor fd = + Descriptors.FieldDescriptor fd = ((Message) obj).getDescriptorForType().findFieldByName(name); + if (fd == null) { + fd = ((Message) obj).getDescriptorForType() + .findFieldByName(camelToSnake(name)); + } if (fd != null) { return ((Message) obj).getField(fd); } } if (obj instanceof Message.Builder) { - final Descriptors.FieldDescriptor fd = + Descriptors.FieldDescriptor fd = ((Message.Builder) obj).getDescriptorForType().findFieldByName(name); + if (fd == null) { + fd = ((Message.Builder) obj).getDescriptorForType() + .findFieldByName(camelToSnake(name)); + } if (fd != null) { return ((Message.Builder) obj).getField(fd); } } return null; } + + private static String camelToSnake(final String name) { + final StringBuilder sb = new StringBuilder(); + for (int i = 0; i < name.length(); i++) { + final char c = name.charAt(i); + if (Character.isUpperCase(c)) { + if (i > 0) { + sb.append('_'); + } + sb.append(Character.toLowerCase(c)); + } else { + sb.append(c); + } + } + return sb.toString(); + } } } diff --git a/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/compiler/LALExpressionExecutionTest.java b/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/compiler/LALExpressionExecutionTest.java new file mode 100644 index 0000000000..30ed016d2c --- /dev/null +++ b/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/compiler/LALExpressionExecutionTest.java @@ -0,0 +1,478 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.log.analyzer.compiler; + +import java.io.File; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import com.google.protobuf.Message; +import com.google.protobuf.util.JsonFormat; +import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; +import org.apache.skywalking.apm.network.logging.v3.JSONLog; +import org.apache.skywalking.apm.network.logging.v3.LogData; +import org.apache.skywalking.apm.network.logging.v3.LogDataBody; +import org.apache.skywalking.apm.network.logging.v3.LogTags; +import org.apache.skywalking.apm.network.logging.v3.TextLog; +import org.apache.skywalking.oap.log.analyzer.dsl.Binding; +import org.apache.skywalking.oap.log.analyzer.dsl.LalExpression; +import org.apache.skywalking.oap.log.analyzer.dsl.spec.filter.FilterSpec; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.SampledTraceBuilder; +import org.apache.skywalking.oap.log.analyzer.module.LogAnalyzerModule; +import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig; +import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleProvider; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.config.NamingControl; +import org.apache.skywalking.oap.server.core.config.group.EndpointNameGrouping; +import org.apache.skywalking.oap.server.core.config.ConfigService; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.ModuleProviderHolder; +import org.apache.skywalking.oap.server.library.module.ModuleServiceHolder; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestFactory; +import org.yaml.snakeyaml.Yaml; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Data-driven runtime execution tests for compiled LAL expressions. + * + * <p>Loads LAL rules from {@code .yaml} files and mock input from + * corresponding {@code .input.data} files in the {@code test-lal/} + * directory tree. For each rule that has a matching input entry, + * compiles the DSL via {@link LALClassGenerator}, executes it against + * a real {@link FilterSpec} + {@link Binding}, and asserts on the + * expected state defined in the {@code expect} block. + */ +class LALExpressionExecutionTest { + + @TestFactory + Collection<DynamicTest> lalExecutionTests() throws Exception { + final List<DynamicTest> tests = new ArrayList<>(); + final FilterSpec filterSpec = buildFilterSpec(); + final LALClassGenerator generator = new LALClassGenerator(); + final Yaml yaml = new Yaml(); + + final Path testLalDir = findTestLalDir(); + if (testLalDir == null) { + return tests; + } + + // Scan subdirectories (oap-cases/, feature-cases/) + final File[] subdirs = testLalDir.toFile().listFiles(File::isDirectory); + if (subdirs == null) { + return tests; + } + + for (final File subdir : subdirs) { + final File[] files = subdir.listFiles(); + if (files == null) { + continue; + } + for (final File yamlFile : files) { + if (!yamlFile.getName().endsWith(".yaml") + && !yamlFile.getName().endsWith(".yml")) { + continue; + } + + // Look for matching .input.data file + final String baseName = yamlFile.getName() + .replaceAll("\\.(yaml|yml)$", ""); + final File inputDataFile = new File(subdir, + baseName + ".input.data"); + if (!inputDataFile.exists()) { + continue; + } + + // Parse the YAML rules + final String yamlContent = + Files.readString(yamlFile.toPath()); + final Map<String, Object> config = yaml.load(yamlContent); + if (config == null || !config.containsKey("rules")) { + continue; + } + @SuppressWarnings("unchecked") + final List<Map<String, String>> rules = + (List<Map<String, String>>) config.get("rules"); + if (rules == null) { + continue; + } + + // Parse the input data + final String inputContent = + Files.readString(inputDataFile.toPath()); + @SuppressWarnings("unchecked") + final Map<String, Map<String, Object>> inputData = + yaml.load(inputContent); + if (inputData == null) { + continue; + } + + final String category = subdir.getName(); + for (final Map<String, String> rule : rules) { + final String ruleName = rule.get("name"); + final String dsl = rule.get("dsl"); + final String ruleLayer = rule.get("layer"); + if (ruleName == null || dsl == null) { + continue; + } + final Map<String, Object> input = + inputData.get(ruleName); + if (input == null) { + continue; + } + + tests.add(DynamicTest.dynamicTest( + category + "/" + baseName + " | " + ruleName, + () -> executeAndAssert( + generator, filterSpec, ruleName, + dsl, ruleLayer, input) + )); + } + } + } + return tests; + } + + private void executeAndAssert( + final LALClassGenerator generator, + final FilterSpec filterSpec, + final String ruleName, + final String dsl, + final String ruleLayer, + final Map<String, Object> input) throws Exception { + final LalExpression expr = generator.compile(dsl); + final LogData.Builder logData = buildLogData(input); + if (ruleLayer != null) { + logData.setLayer(ruleLayer); + } + final Binding binding = new Binding(); + binding.log(logData); + + // Set proto extraLog if specified + final Message extraLog = buildExtraLog(input); + if (extraLog != null) { + binding.extraLog(extraLog); + } + + filterSpec.bind(binding); + expr.execute(filterSpec, binding); + + // Assert expected values + @SuppressWarnings("unchecked") + final Map<String, Object> expect = + (Map<String, Object>) input.get("expect"); + if (expect == null) { + return; + } + + for (final Map.Entry<String, Object> entry : expect.entrySet()) { + final String key = entry.getKey(); + final String expected = String.valueOf(entry.getValue()); + + switch (key) { + case "service": + assertEquals(expected, binding.log().getService(), + ruleName + ": service mismatch"); + break; + case "instance": + assertEquals(expected, + binding.log().getServiceInstance(), + ruleName + ": serviceInstance mismatch"); + break; + case "endpoint": + assertEquals(expected, binding.log().getEndpoint(), + ruleName + ": endpoint mismatch"); + break; + case "layer": + assertEquals(expected, binding.log().getLayer(), + ruleName + ": layer mismatch"); + break; + case "save": + assertEquals(Boolean.parseBoolean(expected), + binding.shouldSave(), + ruleName + ": shouldSave mismatch"); + break; + case "abort": + assertEquals(Boolean.parseBoolean(expected), + binding.shouldAbort(), + ruleName + ": shouldAbort mismatch"); + break; + case "timestamp": + assertEquals(Long.parseLong(expected), + binding.log().getTimestamp(), + ruleName + ": timestamp mismatch"); + break; + default: + if (key.startsWith("sampledTrace.")) { + assertSampledTrace( + ruleName, key, expected, binding); + } else if (key.startsWith("tag.")) { + final String tagKey = key.substring(4); + final List<KeyStringValuePair> tags = + binding.log().getTags().getDataList(); + assertTrue(tags.stream().anyMatch( + t -> tagKey.equals(t.getKey()) + && expected.equals(t.getValue())), + ruleName + ": expected tag " + + tagKey + "=" + expected + + ", got: " + tags.stream() + .map(t -> t.getKey() + "=" + t.getValue()) + .collect(Collectors.joining(", "))); + } + break; + } + } + } + + // ==================== SampledTrace assertions ==================== + + private static void assertSampledTrace( + final String ruleName, + final String key, + final String expected, + final Binding binding) { + final SampledTraceBuilder builder = + binding.sampledTraceBuilder(); + assertTrue(builder != null, + ruleName + ": sampledTraceBuilder is null" + + " but expected " + key + "=" + expected); + + final String field = key.substring("sampledTrace.".length()); + switch (field) { + case "latency": + assertEquals(Long.parseLong(expected), + builder.getLatency(), + ruleName + ": sampledTrace.latency mismatch"); + break; + case "uri": + assertEquals(expected, builder.getUri(), + ruleName + ": sampledTrace.uri mismatch"); + break; + case "reason": + assertEquals(expected, + builder.getReason().name(), + ruleName + ": sampledTrace.reason mismatch"); + break; + case "processId": + assertEquals(expected, builder.getProcessId(), + ruleName + ": sampledTrace.processId mismatch"); + break; + case "destProcessId": + assertEquals(expected, builder.getDestProcessId(), + ruleName + ": sampledTrace.destProcessId mismatch"); + break; + case "detectPoint": + assertEquals(expected, + builder.getDetectPoint().name(), + ruleName + ": sampledTrace.detectPoint mismatch"); + break; + case "componentId": + assertEquals(Integer.parseInt(expected), + builder.getComponentId(), + ruleName + + ": sampledTrace.componentId mismatch"); + break; + default: + throw new IllegalArgumentException( + ruleName + ": unknown sampledTrace field: " + + field); + } + } + + // ==================== LogData builder ==================== + + @SuppressWarnings("unchecked") + private static LogData.Builder buildLogData( + final Map<String, Object> input) { + final LogData.Builder builder = LogData.newBuilder(); + + final String service = (String) input.get("service"); + if (service != null) { + builder.setService(service); + } + + final String instance = (String) input.get("instance"); + if (instance != null) { + builder.setServiceInstance(instance); + } + + final String traceId = (String) input.get("trace-id"); + if (traceId != null) { + builder.setTraceContext( + org.apache.skywalking.apm.network.logging.v3.TraceContext + .newBuilder().setTraceId(traceId)); + } + + final Object tsObj = input.get("timestamp"); + if (tsObj != null) { + builder.setTimestamp(Long.parseLong(String.valueOf(tsObj))); + } + + final String bodyType = (String) input.get("body-type"); + final String body = (String) input.get("body"); + + if ("json".equals(bodyType) && body != null) { + builder.setBody(LogDataBody.newBuilder() + .setJson(JSONLog.newBuilder().setJson(body))); + } else if ("text".equals(bodyType) && body != null) { + builder.setBody(LogDataBody.newBuilder() + .setText(TextLog.newBuilder().setText(body))); + } + + final Map<String, String> tags = + (Map<String, String>) input.get("tags"); + if (tags != null && !tags.isEmpty()) { + final LogTags.Builder tagsBuilder = LogTags.newBuilder(); + for (final Map.Entry<String, String> tag : tags.entrySet()) { + tagsBuilder.addData(KeyStringValuePair.newBuilder() + .setKey(tag.getKey()) + .setValue(tag.getValue())); + } + builder.setTags(tagsBuilder); + } + + return builder; + } + + // ==================== Proto extraLog builder ==================== + + @SuppressWarnings("unchecked") + private static Message buildExtraLog( + final Map<String, Object> input) throws Exception { + final Map<String, String> extraLog = + (Map<String, String>) input.get("extra-log"); + if (extraLog == null) { + return null; + } + + final String protoClass = extraLog.get("proto-class"); + final String protoJson = extraLog.get("proto-json"); + if (protoClass == null || protoJson == null) { + return null; + } + + final Class<?> clazz = Class.forName(protoClass); + final Message.Builder builder = (Message.Builder) + clazz.getMethod("newBuilder").invoke(null); + JsonFormat.parser() + .ignoringUnknownFields() + .merge(protoJson, builder); + return builder.build(); + } + + // ==================== FilterSpec setup ==================== + + private FilterSpec buildFilterSpec() throws Exception { + final ModuleManager manager = mock(ModuleManager.class); + setInternalField(manager, "isInPrepareStage", false); + + when(manager.find(anyString())) + .thenReturn(mock(ModuleProviderHolder.class)); + + final ModuleProviderHolder logHolder = + mock(ModuleProviderHolder.class); + final LogAnalyzerModuleProvider logProvider = + mock(LogAnalyzerModuleProvider.class); + when(logProvider.getMetricConverts()) + .thenReturn(Collections.emptyList()); + when(logHolder.provider()).thenReturn(logProvider); + when(manager.find(LogAnalyzerModule.NAME)).thenReturn(logHolder); + + final ModuleProviderHolder coreHolder = + mock(ModuleProviderHolder.class); + final ModuleServiceHolder coreServices = + mock(ModuleServiceHolder.class); + when(coreHolder.provider()).thenReturn(coreServices); + when(manager.find(CoreModule.NAME)).thenReturn(coreHolder); + + when(coreServices.getService(SourceReceiver.class)) + .thenReturn(mock(SourceReceiver.class)); + when(coreServices.getService(NamingControl.class)) + .thenReturn(new NamingControl( + 200, 200, 200, new EndpointNameGrouping())); + final ConfigService configService = mock(ConfigService.class); + when(configService.getSearchableLogsTags()).thenReturn(""); + when(coreServices.getService(ConfigService.class)) + .thenReturn(configService); + + final FilterSpec filterSpec = + new FilterSpec(manager, new LogAnalyzerModuleConfig()); + setInternalField(filterSpec, "sinkListenerFactories", + Collections.emptyList()); + + return filterSpec; + } + + // ==================== Directory resolution ==================== + + private Path findTestLalDir() { + final String[] candidates = { + // From repo root (e.g., running with -pl from top level) + "test/script-cases/scripts/lal/test-lal", + // From oap-server/analyzer/log-analyzer/ module directory + "../../../test/script-cases/scripts/lal/test-lal", + // From script-runtime-with-groovy checker location + "../../scripts/lal/test-lal" + }; + for (final String candidate : candidates) { + final Path path = Path.of(candidate); + if (Files.isDirectory(path)) { + return path; + } + } + return null; + } + + // ==================== Reflection helpers ==================== + + private static void setInternalField(final Object target, + final String fieldName, + final Object value) { + try { + Field field = null; + Class<?> clazz = target.getClass(); + while (clazz != null && field == null) { + try { + field = clazz.getDeclaredField(fieldName); + } catch (NoSuchFieldException e) { + clazz = clazz.getSuperclass(); + } + } + if (field != null) { + field.setAccessible(true); + field.set(target, value); + } + } catch (Exception e) { + throw new RuntimeException( + "Failed to set field " + fieldName, e); + } + } +} diff --git a/oap-server/analyzer/meter-analyzer/src/main/antlr4/org/apache/skywalking/mal/rt/grammar/MALParser.g4 b/oap-server/analyzer/meter-analyzer/src/main/antlr4/org/apache/skywalking/mal/rt/grammar/MALParser.g4 index 0306ec4a2b..fac507dc01 100644 --- a/oap-server/analyzer/meter-analyzer/src/main/antlr4/org/apache/skywalking/mal/rt/grammar/MALParser.g4 +++ b/oap-server/analyzer/meter-analyzer/src/main/antlr4/org/apache/skywalking/mal/rt/grammar/MALParser.g4 @@ -89,6 +89,8 @@ argument : additiveExpression // nested expression (metric ref, number, arithmetic) | stringList // ["tag1", "tag2"] | numberList // [50, 75, 90, 95, 99] + | L_PAREN stringList R_PAREN // (["tag1", "tag2"]) — extra parens + | L_PAREN numberList R_PAREN // ([50, 75, 90]) — extra parens | closureExpression // {tags -> ...} | enumRef // Layer.GENERAL, K8sRetagType.Pod2Service | STRING // "PT1M", "k8s-key" @@ -137,10 +139,17 @@ closureBody closureStatement : ifStatement | returnStatement + | variableDeclaration | assignmentStatement | expressionStatement ; +// ==================== Variable declarations ==================== +// Groovy-style: String result = "", String protocol = tags['protocol'] +variableDeclaration + : IDENTIFIER IDENTIFIER ASSIGN closureExpr SEMI? + ; + // ==================== Closure statements ==================== ifStatement @@ -193,7 +202,9 @@ closureConditionPrimary ; closureExpr - : closureExpr PLUS closureExpr # closureAdd + : closureExpr QUESTION closureExpr COLON closureExpr # closureTernary + | closureExpr QUESTION COLON closureExpr # closureElvis + | closureExpr PLUS closureExpr # closureAdd | closureExpr MINUS closureExpr # closureSub | closureExpr STAR closureExpr # closureMul | closureExpr SLASH closureExpr # closureDiv @@ -205,11 +216,28 @@ closureExprPrimary | NUMBER # closureNumber | NULL # closureNull | boolLiteral # closureBool + | closureMapLiteral # closureMap | closureMethodChain # closureChain + | L_PAREN closureExpr R_PAREN # closureParen + ; + +// Groovy map literal: ['key': expr, 'key2': expr2] +closureMapLiteral + : L_BRACKET closureMapEntry (COMMA closureMapEntry)* R_BRACKET + ; + +closureMapEntry + : STRING COLON closureExpr ; closureMethodChain - : closureTarget (DOT closureChainSegment)* (safeNav closureChainSegment)* + : closureTarget closureChainAccess* + ; + +closureChainAccess + : DOT closureChainSegment + | safeNav closureChainSegment + | L_BRACKET closureExpr R_BRACKET // direct bracket: tags['key'] ; closureTarget diff --git a/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/compiler/MALClassGenerator.java b/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/compiler/MALClassGenerator.java index e2f10c7966..81aea84996 100644 --- a/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/compiler/MALClassGenerator.java +++ b/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/compiler/MALClassGenerator.java @@ -61,6 +61,11 @@ public final class MALClassGenerator { */ private static final java.util.Map<String, String> ENUM_FQCN; + /** + * Well-known helper classes used inside MAL closures (Groovy imports). + */ + private static final java.util.Map<String, String> CLOSURE_CLASS_FQCN; + static { ENUM_FQCN = new java.util.HashMap<>(); ENUM_FQCN.put("Layer", "org.apache.skywalking.oap.server.core.analysis.Layer"); @@ -69,13 +74,25 @@ public final class MALClassGenerator { "org.apache.skywalking.oap.meter.analyzer.dsl.tagOpt.K8sRetagType"); ENUM_FQCN.put("DownsamplingType", "org.apache.skywalking.oap.meter.analyzer.dsl.DownsamplingType"); + + CLOSURE_CLASS_FQCN = new java.util.HashMap<>(); + CLOSURE_CLASS_FQCN.put("ProcessRegistry", + "org.apache.skywalking.oap.meter.analyzer.dsl.registry.ProcessRegistry"); } private final ClassPool classPool; private int closureCounter; public MALClassGenerator() { - this(ClassPool.getDefault()); + this(createClassPool()); + } + + private static ClassPool createClassPool() { + final ClassPool pool = new ClassPool(true); + pool.appendClassPath( + new javassist.LoaderClassPath( + Thread.currentThread().getContextClassLoader())); + return pool; } public MALClassGenerator(final ClassPool classPool) { @@ -253,28 +270,41 @@ public final class MALClassGenerator { collectClosuresFromChain( ((MALExpressionModel.ParenChainExpr) expr).getMethodChain(), closures); } else if (expr instanceof MALExpressionModel.FunctionCallExpr) { - collectClosuresFromArgs( - ((MALExpressionModel.FunctionCallExpr) expr).getArguments(), closures); + final MALExpressionModel.FunctionCallExpr fce = + (MALExpressionModel.FunctionCallExpr) expr; + collectClosuresFromArgs(fce.getFunctionName(), + fce.getArguments(), closures); collectClosuresFromChain( - ((MALExpressionModel.FunctionCallExpr) expr).getMethodChain(), closures); + fce.getMethodChain(), closures); } } private void collectClosuresFromChain(final List<MALExpressionModel.MethodCall> chain, final List<ClosureInfo> closures) { for (final MALExpressionModel.MethodCall mc : chain) { - collectClosuresFromArgs(mc.getArguments(), closures); + collectClosuresFromArgs(mc.getName(), mc.getArguments(), closures); } } - private void collectClosuresFromArgs(final List<MALExpressionModel.Argument> args, + private void collectClosuresFromArgs(final String methodName, + final List<MALExpressionModel.Argument> args, final List<ClosureInfo> closures) { for (final MALExpressionModel.Argument arg : args) { if (arg instanceof MALExpressionModel.ClosureArgument) { + final String interfaceType; + if ("forEach".equals(methodName)) { + interfaceType = "org.apache.skywalking.oap.meter.analyzer.dsl" + + ".SampleFamilyFunctions$ForEachFunction"; + } else if ("instance".equals(methodName)) { + interfaceType = "org.apache.skywalking.oap.meter.analyzer.dsl" + + ".SampleFamilyFunctions$PropertiesExtractor"; + } else { + interfaceType = "org.apache.skywalking.oap.meter.analyzer.dsl" + + ".SampleFamilyFunctions$TagFunction"; + } final ClosureInfo info = new ClosureInfo( (MALExpressionModel.ClosureArgument) arg, - "org.apache.skywalking.oap.meter.analyzer.dsl" - + ".SampleFamilyFunctions$TagFunction"); + interfaceType); info.fieldIndex = closures.size(); closures.add(info); } else if (arg instanceof MALExpressionModel.ExprArgument) { @@ -284,6 +314,12 @@ public final class MALClassGenerator { } } + private static final String FOR_EACH_FUNCTION_TYPE = + "org.apache.skywalking.oap.meter.analyzer.dsl.SampleFamilyFunctions$ForEachFunction"; + + private static final String PROPERTIES_EXTRACTOR_TYPE = + "org.apache.skywalking.oap.meter.analyzer.dsl.SampleFamilyFunctions$PropertiesExtractor"; + private Object compileClosureClass(final String className, final ClosureInfo info) throws Exception { final CtClass ctClass = classPool.makeClass(className); @@ -292,22 +328,86 @@ public final class MALClassGenerator { final MALExpressionModel.ClosureArgument closure = info.closure; final List<String> params = closure.getParams(); - final String paramName = params.isEmpty() ? "it" : params.get(0); + final boolean isForEach = FOR_EACH_FUNCTION_TYPE.equals(info.interfaceType); + final boolean isPropertiesExtractor = + PROPERTIES_EXTRACTOR_TYPE.equals(info.interfaceType); + + if (isForEach) { + // ForEachFunction: void accept(String element, Map<String, String> tags) + // Closure params: { prefix, tags -> ... } or { element, tags -> ... } + final String elementParam = params.size() >= 1 ? params.get(0) : "element"; + final String tagsParam = params.size() >= 2 ? params.get(1) : "tags"; + + final StringBuilder sb = new StringBuilder(); + sb.append("public void accept(String ").append(elementParam) + .append(", java.util.Map ").append(tagsParam).append(") {\n"); + for (final MALExpressionModel.ClosureStatement stmt : closure.getBody()) { + generateClosureStatement(sb, stmt, tagsParam); + } + sb.append("}\n"); - final StringBuilder sb = new StringBuilder(); - sb.append("public java.util.Map apply(java.util.Map ").append(paramName) - .append(") {\n"); - for (final MALExpressionModel.ClosureStatement stmt : closure.getBody()) { - generateClosureStatement(sb, stmt, paramName); - } - sb.append(" return ").append(paramName).append(";\n"); - sb.append("}\n"); + if (log.isDebugEnabled()) { + log.debug("ForEach closure body:\n{}", sb); + } + ctClass.addMethod(CtNewMethod.make(sb.toString(), ctClass)); + } else if (isPropertiesExtractor) { + // PropertiesExtractor: Map<String,String> apply(Map<String,String> tags) + // Body is typically a single map literal expression + final String paramName = params.isEmpty() ? "it" : params.get(0); + + final StringBuilder sb = new StringBuilder(); + sb.append("public java.util.Map apply(java.util.Map ").append(paramName) + .append(") {\n"); + + // If the body is a single expression statement with a map literal, + // generate HashMap construction as the return value + final List<MALExpressionModel.ClosureStatement> body = closure.getBody(); + if (body.size() == 1 + && body.get(0) instanceof MALExpressionModel.ClosureExprStatement + && ((MALExpressionModel.ClosureExprStatement) body.get(0)).getExpr() + instanceof MALExpressionModel.ClosureMapLiteral) { + final MALExpressionModel.ClosureMapLiteral mapLit = + (MALExpressionModel.ClosureMapLiteral) + ((MALExpressionModel.ClosureExprStatement) body.get(0)).getExpr(); + sb.append(" java.util.Map _result = new java.util.HashMap();\n"); + for (final MALExpressionModel.MapEntry entry : mapLit.getEntries()) { + sb.append(" _result.put(\"") + .append(escapeJava(entry.getKey())).append("\", "); + generateClosureExpr(sb, entry.getValue(), paramName); + sb.append(");\n"); + } + sb.append(" return _result;\n"); + } else { + for (final MALExpressionModel.ClosureStatement stmt : body) { + generateClosureStatement(sb, stmt, paramName); + } + sb.append(" return ").append(paramName).append(";\n"); + } + sb.append("}\n"); - // Also add the Object apply(Object) bridge method - ctClass.addMethod(CtNewMethod.make(sb.toString(), ctClass)); - ctClass.addMethod(CtNewMethod.make( - "public Object apply(Object o) { return apply((java.util.Map) o); }", - ctClass)); + ctClass.addMethod(CtNewMethod.make(sb.toString(), ctClass)); + ctClass.addMethod(CtNewMethod.make( + "public Object apply(Object o) { return apply((java.util.Map) o); }", + ctClass)); + } else { + // TagFunction: Map<String,String> apply(Map<String,String> tags) + final String paramName = params.isEmpty() ? "it" : params.get(0); + + final StringBuilder sb = new StringBuilder(); + sb.append("public java.util.Map apply(java.util.Map ").append(paramName) + .append(") {\n"); + for (final MALExpressionModel.ClosureStatement stmt : closure.getBody()) { + generateClosureStatement(sb, stmt, paramName); + } + sb.append(" return ").append(paramName).append(";\n"); + sb.append("}\n"); + + // Also add the Object apply(Object) bridge method + ctClass.addMethod(CtNewMethod.make(sb.toString(), ctClass)); + ctClass.addMethod(CtNewMethod.make( + "public Object apply(Object o) { return apply((java.util.Map) o); }", + ctClass)); + } final Class<?> clazz = ctClass.toClass(MalExpressionPackageHolder.class); ctClass.detach(); @@ -427,12 +527,10 @@ public final class MALClassGenerator { sb.append(").multiply(Double.valueOf(").append(num).append("))"); break; case DIV: - sb.append("("); + sb.append("org.apache.skywalking.oap.meter.analyzer.compiler.rt") + .append(".MalRuntimeHelper.divReverse(").append(num).append(", "); generateExpr(sb, right); - sb.append(").newValue(new java.util.function.Function() { ") - .append("public Object apply(Object v) { ") - .append("return Double.valueOf(").append(num) - .append(" / ((Double)v).doubleValue()); } })"); + sb.append(")"); break; default: throw new IllegalArgumentException("Unsupported op: " + op); @@ -463,6 +561,16 @@ public final class MALClassGenerator { "tagEqual", "tagNotEqual", "tagMatch", "tagNotMatch" ); + /** + * Methods on SampleFamily whose first argument is a primitive {@code double}. + * Javassist cannot auto-unbox {@code Double} to {@code double}, so numeric + * arguments to these methods must be emitted as raw double literals. + */ + private static final Set<String> PRIMITIVE_DOUBLE_METHODS = Set.of( + "valueEqual", "valueNotEqual", "valueGreater", + "valueGreaterEqual", "valueLess", "valueLessEqual" + ); + private void generateMethodChain(final StringBuilder sb, final List<MALExpressionModel.MethodCall> chain) { for (final MALExpressionModel.MethodCall mc : chain) { @@ -479,17 +587,41 @@ public final class MALClassGenerator { } sb.append('}'); } else { + final boolean primitiveDouble = + PRIMITIVE_DOUBLE_METHODS.contains(mc.getName()); for (int i = 0; i < args.size(); i++) { if (i > 0) { sb.append(", "); } - generateArgument(sb, args.get(i)); + generateMethodCallArg(sb, args.get(i), primitiveDouble); } } sb.append(')'); } } + /** + * Generates a method call argument, handling numeric ExprArgument + * specially when the target method expects a primitive double. + */ + private void generateMethodCallArg(final StringBuilder sb, + final MALExpressionModel.Argument arg, + final boolean primitiveDouble) { + if (primitiveDouble + && arg instanceof MALExpressionModel.ExprArgument) { + final MALExpressionModel.Expr innerExpr = + ((MALExpressionModel.ExprArgument) arg).getExpr(); + if (innerExpr instanceof MALExpressionModel.NumberExpr) { + // Emit raw double literal for methods taking primitive double + final double num = + ((MALExpressionModel.NumberExpr) innerExpr).getValue(); + sb.append(num); + return; + } + } + generateArgument(sb, arg); + } + private static boolean allStringArgs(final List<MALExpressionModel.Argument> args) { for (final MALExpressionModel.Argument arg : args) { if (!(arg instanceof MALExpressionModel.StringArgument)) { @@ -547,7 +679,12 @@ public final class MALClassGenerator { } else if (arg instanceof MALExpressionModel.ExprArgument) { final MALExpressionModel.Expr innerExpr = ((MALExpressionModel.ExprArgument) arg).getExpr(); - if (innerExpr instanceof MALExpressionModel.MetricExpr + if (innerExpr instanceof MALExpressionModel.NumberExpr) { + // Numeric literal argument (e.g., valueEqual(1), multiply(100)) + // Emit as Double.valueOf() to match Number parameter types. + final double num = ((MALExpressionModel.NumberExpr) innerExpr).getValue(); + sb.append("Double.valueOf(").append(num).append(")"); + } else if (innerExpr instanceof MALExpressionModel.MetricExpr && ((MALExpressionModel.MetricExpr) innerExpr).getMethodChain().isEmpty()) { // Bare identifier — could be an enum constant like SUM, AVG final String name = @@ -578,8 +715,9 @@ public final class MALClassGenerator { if (stmt instanceof MALExpressionModel.ClosureAssignment) { final MALExpressionModel.ClosureAssignment assign = (MALExpressionModel.ClosureAssignment) stmt; - sb.append(" ").append(paramName).append(".put(\"") - .append(escapeJava(assign.getTarget())).append("\", "); + sb.append(" ").append(assign.getMapVar()).append(".put("); + generateClosureExpr(sb, assign.getKeyExpr(), paramName); + sb.append(", "); generateClosureExpr(sb, assign.getValue(), paramName); sb.append(");\n"); } else if (stmt instanceof MALExpressionModel.ClosureIfStatement) { @@ -600,9 +738,28 @@ public final class MALClassGenerator { sb.append(" }\n"); } } else if (stmt instanceof MALExpressionModel.ClosureReturnStatement) { - sb.append(" return (java.util.Map) "); - generateClosureExpr(sb, - ((MALExpressionModel.ClosureReturnStatement) stmt).getValue(), paramName); + final MALExpressionModel.ClosureReturnStatement retStmt = + (MALExpressionModel.ClosureReturnStatement) stmt; + if (retStmt.getValue() == null) { + // Bare return (void return for ForEachFunction, or early exit) + sb.append(" return;\n"); + } else { + sb.append(" return (java.util.Map) "); + generateClosureExpr(sb, retStmt.getValue(), paramName); + sb.append(";\n"); + } + } else if (stmt instanceof MALExpressionModel.ClosureVarDecl) { + final MALExpressionModel.ClosureVarDecl vd = + (MALExpressionModel.ClosureVarDecl) stmt; + sb.append(" ").append(vd.getTypeName()).append(" ") + .append(vd.getVarName()).append(" = "); + generateClosureExpr(sb, vd.getInitializer(), paramName); + sb.append(";\n"); + } else if (stmt instanceof MALExpressionModel.ClosureVarAssign) { + final MALExpressionModel.ClosureVarAssign va = + (MALExpressionModel.ClosureVarAssign) stmt; + sb.append(" ").append(va.getVarName()).append(" = "); + generateClosureExpr(sb, va.getValue(), paramName); sb.append(";\n"); } else if (stmt instanceof MALExpressionModel.ClosureExprStatement) { sb.append(" "); @@ -625,6 +782,20 @@ public final class MALClassGenerator { sb.append(((MALExpressionModel.ClosureBoolLiteral) expr).isValue()); } else if (expr instanceof MALExpressionModel.ClosureNullLiteral) { sb.append("null"); + } else if (expr instanceof MALExpressionModel.ClosureMapLiteral) { + // Inline map construction using java.util.Map.of() + final MALExpressionModel.ClosureMapLiteral mapLit = + (MALExpressionModel.ClosureMapLiteral) expr; + sb.append("java.util.Map.of("); + for (int i = 0; i < mapLit.getEntries().size(); i++) { + if (i > 0) { + sb.append(", "); + } + final MALExpressionModel.MapEntry entry = mapLit.getEntries().get(i); + sb.append('"').append(escapeJava(entry.getKey())).append("\", "); + generateClosureExpr(sb, entry.getValue(), paramName); + } + sb.append(")"); } else if (expr instanceof MALExpressionModel.ClosureMethodChain) { generateClosureMethodChain(sb, (MALExpressionModel.ClosureMethodChain) expr, paramName); @@ -651,6 +822,24 @@ public final class MALClassGenerator { } generateClosureExpr(sb, bin.getRight(), paramName); sb.append(")"); + } else if (expr instanceof MALExpressionModel.ClosureTernaryExpr) { + final MALExpressionModel.ClosureTernaryExpr ternary = + (MALExpressionModel.ClosureTernaryExpr) expr; + sb.append("(((Object)("); + generateClosureExpr(sb, ternary.getCondition(), paramName); + sb.append(")) != null ? ("); + generateClosureExpr(sb, ternary.getTrueExpr(), paramName); + sb.append(") : ("); + generateClosureExpr(sb, ternary.getFalseExpr(), paramName); + sb.append("))"); + } else if (expr instanceof MALExpressionModel.ClosureElvisExpr) { + final MALExpressionModel.ClosureElvisExpr elvis = + (MALExpressionModel.ClosureElvisExpr) expr; + sb.append("java.util.Optional.ofNullable("); + generateClosureExpr(sb, elvis.getPrimary(), paramName); + sb.append(").orElse("); + generateClosureExpr(sb, elvis.getFallback(), paramName); + sb.append(")"); } } @@ -660,31 +849,70 @@ public final class MALClassGenerator { final String paramName) { // tags.key -> tags.get("key") // tags['key'] -> tags.get("key") + final String target = chain.getTarget(); + final String resolvedTarget = CLOSURE_CLASS_FQCN.getOrDefault(target, target); + final boolean isClassRef = CLOSURE_CLASS_FQCN.containsKey(target); final List<MALExpressionModel.ClosureChainSegment> segs = chain.getSegments(); + + // Static class method call: ProcessRegistry.generateVirtualLocalProcess(...) + if (isClassRef) { + final StringBuilder local = new StringBuilder(); + local.append(resolvedTarget); + for (final MALExpressionModel.ClosureChainSegment seg : segs) { + if (seg instanceof MALExpressionModel.ClosureMethodCallSeg) { + final MALExpressionModel.ClosureMethodCallSeg mc = + (MALExpressionModel.ClosureMethodCallSeg) seg; + local.append('.').append(mc.getName()).append('('); + for (int i = 0; i < mc.getArguments().size(); i++) { + if (i > 0) { + local.append(", "); + } + generateClosureExpr(local, mc.getArguments().get(i), paramName); + } + local.append(')'); + } else if (seg instanceof MALExpressionModel.ClosureFieldAccess) { + local.append('.').append( + ((MALExpressionModel.ClosureFieldAccess) seg).getName()); + } + } + sb.append(local); + return; + } + + if (segs.isEmpty()) { + // Bare identifier (e.g. a local variable like "prefix", "result") + sb.append(resolvedTarget); + return; + } + if (segs.size() == 1 && segs.get(0) instanceof MALExpressionModel.ClosureFieldAccess) { final String key = ((MALExpressionModel.ClosureFieldAccess) segs.get(0)).getName(); - sb.append(chain.getTarget()).append(".get(\"") + sb.append("(String) ").append(resolvedTarget).append(".get(\"") .append(escapeJava(key)).append("\")"); } else if (segs.size() == 1 && segs.get(0) instanceof MALExpressionModel.ClosureIndexAccess) { - sb.append(chain.getTarget()).append(".get("); + sb.append("(String) ").append(resolvedTarget).append(".get("); generateClosureExpr(sb, ((MALExpressionModel.ClosureIndexAccess) segs.get(0)).getIndex(), paramName); sb.append(")"); } else { // General chain: build in a local buffer to support safe navigation final StringBuilder local = new StringBuilder(); - local.append(chain.getTarget()); + local.append(resolvedTarget); for (final MALExpressionModel.ClosureChainSegment seg : segs) { if (seg instanceof MALExpressionModel.ClosureFieldAccess) { - local.append(".get(\"") + final String prior = local.toString(); + local.setLength(0); + local.append("(String) ").append(prior).append(".get(\"") .append(escapeJava( ((MALExpressionModel.ClosureFieldAccess) seg).getName())) .append("\")"); } else if (seg instanceof MALExpressionModel.ClosureIndexAccess) { - local.append(".get("); + final String prior2 = local.toString(); + local.setLength(0); + local.append("(String) ").append(prior2).append(".get("); generateClosureExpr(local, ((MALExpressionModel.ClosureIndexAccess) seg).getIndex(), paramName); local.append(")"); diff --git a/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/compiler/MALExpressionModel.java b/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/compiler/MALExpressionModel.java index 455b5db178..d6fda7331c 100644 --- a/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/compiler/MALExpressionModel.java +++ b/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/compiler/MALExpressionModel.java @@ -271,13 +271,53 @@ public final class MALExpressionModel { } } + /** + * Assignment statement: {@code tags.key = expr} or {@code tags[expr] = expr} + * + * <p>{@code mapVar} is the variable (e.g. "tags"), {@code keyExpr} is the key + * expression (string literal for field access, or arbitrary expression for bracket access). + */ @Getter public static final class ClosureAssignment implements ClosureStatement { - private final String target; + private final String mapVar; + private final ClosureExpr keyExpr; private final ClosureExpr value; - public ClosureAssignment(final String target, final ClosureExpr value) { - this.target = target; + public ClosureAssignment(final String mapVar, final ClosureExpr keyExpr, + final ClosureExpr value) { + this.mapVar = mapVar; + this.keyExpr = keyExpr; + this.value = value; + } + } + + /** + * Local variable declaration: {@code String result = ""}, {@code String protocol = tags['protocol']} + */ + @Getter + public static final class ClosureVarDecl implements ClosureStatement { + private final String typeName; + private final String varName; + private final ClosureExpr initializer; + + public ClosureVarDecl(final String typeName, final String varName, + final ClosureExpr initializer) { + this.typeName = typeName; + this.varName = varName; + this.initializer = initializer; + } + } + + /** + * Local variable reassignment: {@code result = '129'} + */ + @Getter + public static final class ClosureVarAssign implements ClosureStatement { + private final String varName; + private final ClosureExpr value; + + public ClosureVarAssign(final String varName, final ClosureExpr value) { + this.varName = varName; this.value = value; } } @@ -326,6 +366,32 @@ public final class MALExpressionModel { public static final class ClosureNullLiteral implements ClosureExpr { } + /** + * Groovy map literal: {@code ['pod': tags.pod, 'namespace': tags.namespace]} + */ + @Getter + public static final class ClosureMapLiteral implements ClosureExpr { + private final List<MapEntry> entries; + + public ClosureMapLiteral(final List<MapEntry> entries) { + this.entries = Collections.unmodifiableList(entries); + } + } + + /** + * Single entry in a Groovy map literal: {@code 'key': expr} + */ + @Getter + public static final class MapEntry { + private final String key; + private final ClosureExpr value; + + public MapEntry(final String key, final ClosureExpr value) { + this.key = key; + this.value = value; + } + } + /** * Method chain in closure: {@code tags.service_name}, {@code tags['key']}, * {@code tags.service?.trim()} @@ -357,6 +423,36 @@ public final class MALExpressionModel { } } + @Getter + public static final class ClosureElvisExpr implements ClosureExpr { + private final ClosureExpr primary; + private final ClosureExpr fallback; + + public ClosureElvisExpr(final ClosureExpr primary, + final ClosureExpr fallback) { + this.primary = primary; + this.fallback = fallback; + } + } + + /** + * Ternary expression: {@code condition ? trueExpr : falseExpr} + */ + @Getter + public static final class ClosureTernaryExpr implements ClosureExpr { + private final ClosureExpr condition; + private final ClosureExpr trueExpr; + private final ClosureExpr falseExpr; + + public ClosureTernaryExpr(final ClosureExpr condition, + final ClosureExpr trueExpr, + final ClosureExpr falseExpr) { + this.condition = condition; + this.trueExpr = trueExpr; + this.falseExpr = falseExpr; + } + } + // ==================== Closure chain segments ==================== public interface ClosureChainSegment { diff --git a/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/compiler/MALScriptParser.java b/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/compiler/MALScriptParser.java index 2c21670387..79a7c83139 100644 --- a/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/compiler/MALScriptParser.java +++ b/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/compiler/MALScriptParser.java @@ -51,6 +51,8 @@ import org.apache.skywalking.oap.meter.analyzer.compiler.MALExpressionModel.Clos import org.apache.skywalking.oap.meter.analyzer.compiler.MALExpressionModel.ClosureReturnStatement; import org.apache.skywalking.oap.meter.analyzer.compiler.MALExpressionModel.ClosureStatement; import org.apache.skywalking.oap.meter.analyzer.compiler.MALExpressionModel.ClosureStringLiteral; +import org.apache.skywalking.oap.meter.analyzer.compiler.MALExpressionModel.ClosureVarAssign; +import org.apache.skywalking.oap.meter.analyzer.compiler.MALExpressionModel.ClosureVarDecl; import org.apache.skywalking.oap.meter.analyzer.compiler.MALExpressionModel.CompareOp; import org.apache.skywalking.oap.meter.analyzer.compiler.MALExpressionModel.EnumRefArgument; import org.apache.skywalking.oap.meter.analyzer.compiler.MALExpressionModel.Expr; @@ -302,11 +304,36 @@ public final class MALScriptParser { ? convertClosureExpr(ctx.returnStatement().closureExpr()) : null; return new ClosureReturnStatement(value); } + if (ctx.variableDeclaration() != null) { + final MALParser.VariableDeclarationContext vd = ctx.variableDeclaration(); + return new ClosureVarDecl( + vd.IDENTIFIER(0).getText(), + vd.IDENTIFIER(1).getText(), + convertClosureExpr(vd.closureExpr())); + } if (ctx.assignmentStatement() != null) { - final String target = ctx.assignmentStatement().closureFieldAccess().getText(); + final MALParser.ClosureFieldAccessContext fa = + ctx.assignmentStatement().closureFieldAccess(); + final List<org.antlr.v4.runtime.tree.TerminalNode> ids = fa.IDENTIFIER(); + final String firstId = ids.get(0).getText(); + if (ids.size() == 1 && fa.closureExpr() == null) { + // bare variable assignment: result = '129' + final ClosureExpr value = + convertClosureExpr(ctx.assignmentStatement().closureExpr()); + return new ClosureVarAssign(firstId, value); + } + // Map assignment: tags.field = value or tags[expr] = value + final ClosureExpr keyExpr; + if (fa.closureExpr() != null) { + // tags[expr] = value + keyExpr = convertClosureExpr(fa.closureExpr()); + } else { + // tags.field = value — the key is the last IDENTIFIER + keyExpr = new ClosureStringLiteral(ids.get(ids.size() - 1).getText()); + } final ClosureExpr value = convertClosureExpr(ctx.assignmentStatement().closureExpr()); - return new ClosureAssignment(target, value); + return new ClosureAssignment(firstId, keyExpr, value); } // expressionStatement return new ClosureExprStatement( @@ -421,6 +448,21 @@ public final class MALScriptParser { } private ClosureExpr convertClosureExpr(final MALParser.ClosureExprContext ctx) { + if (ctx instanceof MALParser.ClosureTernaryContext) { + final MALParser.ClosureTernaryContext ternary = + (MALParser.ClosureTernaryContext) ctx; + return new MALExpressionModel.ClosureTernaryExpr( + convertClosureExpr(ternary.closureExpr(0)), + convertClosureExpr(ternary.closureExpr(1)), + convertClosureExpr(ternary.closureExpr(2))); + } + if (ctx instanceof MALParser.ClosureElvisContext) { + final MALParser.ClosureElvisContext elvis = + (MALParser.ClosureElvisContext) ctx; + return new MALExpressionModel.ClosureElvisExpr( + convertClosureExpr(elvis.closureExpr(0)), + convertClosureExpr(elvis.closureExpr(1))); + } if (ctx instanceof MALParser.ClosureAddContext) { final MALParser.ClosureAddContext add = (MALParser.ClosureAddContext) ctx; return new ClosureBinaryExpr( @@ -473,6 +515,22 @@ public final class MALScriptParser { final MALParser.ClosureBoolContext bc = (MALParser.ClosureBoolContext) ctx; return new ClosureBoolLiteral(bc.boolLiteral().TRUE() != null); } + if (ctx instanceof MALParser.ClosureParenContext) { + return convertClosureExpr( + ((MALParser.ClosureParenContext) ctx).closureExpr()); + } + if (ctx instanceof MALParser.ClosureMapContext) { + final MALParser.ClosureMapLiteralContext mapCtx = + ((MALParser.ClosureMapContext) ctx).closureMapLiteral(); + final List<MALExpressionModel.MapEntry> entries = new ArrayList<>(); + for (final MALParser.ClosureMapEntryContext entry : + mapCtx.closureMapEntry()) { + entries.add(new MALExpressionModel.MapEntry( + stripQuotes(entry.STRING().getText()), + convertClosureExpr(entry.closureExpr()))); + } + return new MALExpressionModel.ClosureMapLiteral(entries); + } // closureChain final MALParser.ClosureChainContext chain = (MALParser.ClosureChainContext) ctx; return convertClosureMethodChain(chain.closureMethodChain()); @@ -483,14 +541,16 @@ public final class MALScriptParser { final String target = ctx.closureTarget().IDENTIFIER().getText(); final List<ClosureChainSegment> segments = new ArrayList<>(); - final int totalSegs = ctx.closureChainSegment().size(); - final int safeNavCount = ctx.safeNav().size(); - final int dotSegCount = totalSegs - safeNavCount; - - for (int i = 0; i < totalSegs; i++) { - final boolean isSafeNav = i >= dotSegCount; - segments.add(convertClosureChainSegment( - ctx.closureChainSegment().get(i), isSafeNav)); + for (final MALParser.ClosureChainAccessContext acc : ctx.closureChainAccess()) { + if (acc.closureChainSegment() != null) { + final boolean isSafeNav = acc.safeNav() != null; + segments.add(convertClosureChainSegment( + acc.closureChainSegment(), isSafeNav)); + } else if (acc.closureExpr() != null) { + // Direct bracket access: tags['key'] or tags[expr] + segments.add(new ClosureIndexAccess( + convertClosureExpr(acc.closureExpr()))); + } } return new ClosureMethodChain(target, segments); diff --git a/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/compiler/rt/MalRuntimeHelper.java b/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/compiler/rt/MalRuntimeHelper.java new file mode 100644 index 0000000000..759e6d926c --- /dev/null +++ b/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/compiler/rt/MalRuntimeHelper.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.meter.analyzer.compiler.rt; + +import org.apache.skywalking.oap.meter.analyzer.dsl.Sample; +import org.apache.skywalking.oap.meter.analyzer.dsl.SampleFamily; +import org.apache.skywalking.oap.meter.analyzer.dsl.SampleFamilyBuilder; + +/** + * Static helper methods called by v2-generated {@code MalExpression} classes. + * Keeps new runtime behaviour in the v2 compiler package, avoiding modifications + * to the shared {@link SampleFamily} class. + */ +public final class MalRuntimeHelper { + + private MalRuntimeHelper() { + } + + /** + * Reverse division: computes {@code numerator / v} for each sample value {@code v}. + * Used by generated code for {@code Number / SampleFamily} expressions. + */ + public static SampleFamily divReverse(final double numerator, + final SampleFamily sf) { + if (sf == SampleFamily.EMPTY) { + return SampleFamily.EMPTY; + } + final Sample[] original = sf.samples; + final Sample[] result = new Sample[original.length]; + for (int i = 0; i < original.length; i++) { + result[i] = original[i].toBuilder() + .value(numerator / original[i].getValue()) + .build(); + } + return SampleFamilyBuilder.newBuilder(result).build(); + } +} diff --git a/oap-server/analyzer/meter-analyzer/src/test/java/org/apache/skywalking/oap/meter/analyzer/compiler/MALClassGeneratorTest.java b/oap-server/analyzer/meter-analyzer/src/test/java/org/apache/skywalking/oap/meter/analyzer/compiler/MALClassGeneratorTest.java index 11b96642bb..60496f04f4 100644 --- a/oap-server/analyzer/meter-analyzer/src/test/java/org/apache/skywalking/oap/meter/analyzer/compiler/MALClassGeneratorTest.java +++ b/oap-server/analyzer/meter-analyzer/src/test/java/org/apache/skywalking/oap/meter/analyzer/compiler/MALClassGeneratorTest.java @@ -127,6 +127,24 @@ class MALClassGeneratorTest { + " && tags.Service?.trim() }")); } + @Test + void compileValueEqual() throws Exception { + final MalExpression expr = generator.compile( + "test_value_equal", + "kube_node_status_condition.valueEqual(1).sum(['cluster','node','condition'])"); + assertNotNull(expr); + assertNotNull(expr.run(java.util.Map.of())); + } + + @Test + void compileMethodCallMultiply() throws Exception { + final MalExpression expr = generator.compile( + "test_multiply", + "process_cpu_usage.multiply(100)"); + assertNotNull(expr); + assertNotNull(expr.run(java.util.Map.of())); + } + // ==================== Error handling tests ==================== @Test @@ -166,4 +184,178 @@ class MALClassGeneratorTest { assertThrows(Exception.class, () -> generator.compileFilter("{ }")); } + + // ==================== Closure key extraction tests ==================== + + @Test + void tagClosurePutsCorrectKey() throws Exception { + // Issue: tags.cluster = expr should generate tags.put("cluster", ...) + // NOT tags.put("tags.cluster", ...) + final MalExpression expr = generator.compile( + "test_key", + "metric.tag({tags -> tags.cluster = 'activemq::' + tags.cluster})"); + assertNotNull(expr); + final String source = generator.generateSource( + "metric.tag({tags -> tags.cluster = 'activemq::' + tags.cluster})"); + assertTrue(source.contains("this._closure0"), + "Generated source should reference pre-compiled closure"); + } + + @Test + void tagClosureKeyExtractionViaGeneratedCode() throws Exception { + // Verify the closure generates correct put("cluster", ...) not put("tags.cluster", ...) + final MalExpression expr = generator.compile( + "test_key_gen", + "metric.tag({tags -> tags.service_name = 'svc1'})"); + assertNotNull(expr); + assertNotNull(expr.run(java.util.Map.of())); + } + + @Test + void tagClosureBracketAssignment() throws Exception { + // tags['key_name'] = 'value' should also use correct key + final MalExpression expr = generator.compile( + "test_bracket", + "metric.tag({tags -> tags['my_key'] = 'my_value'})"); + assertNotNull(expr); + assertNotNull(expr.run(java.util.Map.of())); + } + + // ==================== forEach closure tests ==================== + + @Test + void forEachClosureCompiles() throws Exception { + // forEach requires ForEachFunction.accept(String, Map), not TagFunction.apply(Map) + final MalExpression expr = generator.compile( + "test_foreach", + "metric.forEach(['client', 'server'], {prefix, tags ->" + + " tags[prefix + '_name'] = 'value'})"); + assertNotNull(expr); + assertNotNull(expr.run(java.util.Map.of())); + } + + @Test + void forEachClosureWithBareReturn() throws Exception { + // forEach with bare return (void method) — should not throw + final MalExpression expr = generator.compile( + "test_foreach_return", + "metric.forEach(['x'], {prefix, tags ->\n" + + " if (tags[prefix + '_id'] != null) {\n" + + " return\n" + + " }\n" + + " tags[prefix + '_id'] = 'default'\n" + + "})"); + assertNotNull(expr); + assertNotNull(expr.run(java.util.Map.of())); + } + + @Test + void forEachClosureWithVarDeclAndElseIf() throws Exception { + // Full pattern from network-profiling.yaml second closure + final MalExpression expr = generator.compile( + "test_foreach_vars", + "metric.forEach(['component'], {key, tags ->\n" + + " String result = \"\"\n" + + " String protocol = tags['protocol']\n" + + " String ssl = tags['is_ssl']\n" + + " if (protocol == 'http' && ssl == 'true') {\n" + + " result = '129'\n" + + " } else if (protocol == 'http') {\n" + + " result = '49'\n" + + " } else if (ssl == 'true') {\n" + + " result = '130'\n" + + " } else {\n" + + " result = '110'\n" + + " }\n" + + " tags[key] = result\n" + + "})"); + assertNotNull(expr); + assertNotNull(expr.run(java.util.Map.of())); + } + + // ==================== ProcessRegistry FQCN resolution tests ==================== + + @Test + void processRegistryResolvedToFQCN() throws Exception { + // ProcessRegistry.generateVirtualLocalProcess() should resolve to FQCN + final MalExpression expr = generator.compile( + "test_registry", + "metric.forEach(['client'], {prefix, tags ->\n" + + " tags[prefix + '_process_id'] = " + + "ProcessRegistry.generateVirtualLocalProcess(tags.service, tags.instance)\n" + + "})"); + assertNotNull(expr); + // We can't easily execute this (needs ProcessRegistry runtime) but compile should succeed + } + + // ==================== Network-profiling full expression tests ==================== + + @Test + void networkProfilingFirstClosureCompiles() throws Exception { + // Full first closure from network-profiling.yaml expPrefix + final MalExpression expr = generator.compile( + "test_np1", + "metric.forEach(['client', 'server'], { prefix, tags ->\n" + + " if (tags[prefix + '_process_id'] != null) {\n" + + " return\n" + + " }\n" + + " if (tags[prefix + '_local'] == 'true') {\n" + + " tags[prefix + '_process_id'] = ProcessRegistry" + + ".generateVirtualLocalProcess(tags.service, tags.instance)\n" + + " return\n" + + " }\n" + + " tags[prefix + '_process_id'] = ProcessRegistry" + + ".generateVirtualRemoteProcess(tags.service, tags.instance," + + " tags[prefix + '_address'])\n" + + " })"); + assertNotNull(expr); + } + + @Test + void networkProfilingSecondClosureCompiles() throws Exception { + // Full second closure from network-profiling.yaml expPrefix + final MalExpression expr = generator.compile( + "test_np2", + "metric.forEach(['component'], { key, tags ->\n" + + " String result = \"\"\n" + + " // protocol are defined in the component-libraries.yml\n" + + " String protocol = tags['protocol']\n" + + " String ssl = tags['is_ssl']\n" + + " if (protocol == 'http' && ssl == 'true') {\n" + + " result = '129'\n" + + " } else if (protocol == 'http') {\n" + + " result = '49'\n" + + " } else if (ssl == 'true') {\n" + + " result = '130'\n" + + " } else {\n" + + " result = '110'\n" + + " }\n" + + " tags[key] = result\n" + + " })"); + assertNotNull(expr); + } + + // ==================== String concatenation in closures ==================== + + @Test + void apisixExpressionCompiles() throws Exception { + // The APISIX expression that originally triggered the E2E failure: + // safe navigation + elvis + bracket access + string concat + final MalExpression expr = generator.compile( + "test_apisix", + "metric.tag({tags -> tags.service_name = 'APISIX::'" + + "+(tags['skywalking_service']?.trim()?:'APISIX')})"); + assertNotNull(expr); + assertNotNull(expr.run(java.util.Map.of())); + } + + @Test + void closureStringConcatenation() throws Exception { + // APISIX-style: tags.service_name = 'APISIX::' + tags.service + final MalExpression expr = generator.compile( + "test_concat", + "metric.tag({tags -> tags.service_name = 'APISIX::' + tags.service})"); + assertNotNull(expr); + assertNotNull(expr.run(java.util.Map.of())); + } } diff --git a/oap-server/analyzer/meter-analyzer/src/test/java/org/apache/skywalking/oap/meter/analyzer/compiler/MALScriptParserTest.java b/oap-server/analyzer/meter-analyzer/src/test/java/org/apache/skywalking/oap/meter/analyzer/compiler/MALScriptParserTest.java index 40f50f5745..eeb71518c7 100644 --- a/oap-server/analyzer/meter-analyzer/src/test/java/org/apache/skywalking/oap/meter/analyzer/compiler/MALScriptParserTest.java +++ b/oap-server/analyzer/meter-analyzer/src/test/java/org/apache/skywalking/oap/meter/analyzer/compiler/MALScriptParserTest.java @@ -230,6 +230,127 @@ class MALScriptParserTest { assertEquals("Pod2Service", enumArg.getEnumValue()); } + @Test + void parseTagAssignmentExtractsCorrectKey() { + // Issue: tags.cluster = expr should produce key "cluster", not "tags.cluster" + final MALExpressionModel.Expr ast = MALScriptParser.parse( + "metric.tag({tags -> tags.cluster = 'activemq::' + tags.cluster})"); + assertInstanceOf(MetricExpr.class, ast); + final MetricExpr metric = (MetricExpr) ast; + final ClosureArgument closure = + (ClosureArgument) metric.getMethodChain().get(0).getArguments().get(0); + assertEquals(1, closure.getBody().size()); + + final MALExpressionModel.ClosureAssignment assign = + (MALExpressionModel.ClosureAssignment) closure.getBody().get(0); + assertEquals("tags", assign.getMapVar()); + // Key should be "cluster", not "tags.cluster" + assertInstanceOf(MALExpressionModel.ClosureStringLiteral.class, assign.getKeyExpr()); + assertEquals("cluster", + ((MALExpressionModel.ClosureStringLiteral) assign.getKeyExpr()).getValue()); + } + + @Test + void parseTagBracketAssignment() { + // tags[prefix + '_process_id'] = expr + final MALExpressionModel.Expr ast = MALScriptParser.parse( + "metric.tag({prefix, tags -> tags[prefix + '_id'] = 'val'})"); + assertInstanceOf(MetricExpr.class, ast); + final MetricExpr metric = (MetricExpr) ast; + final ClosureArgument closure = + (ClosureArgument) metric.getMethodChain().get(0).getArguments().get(0); + assertEquals(List.of("prefix", "tags"), closure.getParams()); + + final MALExpressionModel.ClosureAssignment assign = + (MALExpressionModel.ClosureAssignment) closure.getBody().get(0); + assertEquals("tags", assign.getMapVar()); + // Key is a binary expression (prefix + '_id') + assertInstanceOf(MALExpressionModel.ClosureBinaryExpr.class, assign.getKeyExpr()); + } + + @Test + void parseForEachClosure() { + final MALExpressionModel.Expr ast = MALScriptParser.parse( + "metric.forEach(['client', 'server'], {prefix, tags -> tags.key = prefix})"); + assertInstanceOf(MetricExpr.class, ast); + final MetricExpr metric = (MetricExpr) ast; + assertEquals("forEach", metric.getMethodChain().get(0).getName()); + + final ClosureArgument closure = + (ClosureArgument) metric.getMethodChain().get(0).getArguments().get(1); + assertEquals(List.of("prefix", "tags"), closure.getParams()); + } + + @Test + void parseVariableDeclaration() { + // String result = "" — Groovy local variable declaration + final MALExpressionModel.Expr ast = MALScriptParser.parse( + "metric.forEach(['x'], {key, tags ->\n" + + " String result = \"\"\n" + + " tags[key] = result\n" + + "})"); + assertInstanceOf(MetricExpr.class, ast); + final MetricExpr metric = (MetricExpr) ast; + final ClosureArgument closure = + (ClosureArgument) metric.getMethodChain().get(0).getArguments().get(1); + assertEquals(2, closure.getBody().size()); + // First statement: variable declaration + assertInstanceOf(MALExpressionModel.ClosureVarDecl.class, closure.getBody().get(0)); + final MALExpressionModel.ClosureVarDecl vd = + (MALExpressionModel.ClosureVarDecl) closure.getBody().get(0); + assertEquals("String", vd.getTypeName()); + assertEquals("result", vd.getVarName()); + } + + @Test + void parseBareReturn() { + // return with no expression (void return) + final MALExpressionModel.Expr ast = MALScriptParser.parse( + "metric.forEach(['x'], {prefix, tags ->\n" + + " if (tags[prefix + '_id'] != null) {\n" + + " return\n" + + " }\n" + + " tags[prefix + '_id'] = 'default'\n" + + "})"); + assertInstanceOf(MetricExpr.class, ast); + final MetricExpr metric = (MetricExpr) ast; + final ClosureArgument closure = + (ClosureArgument) metric.getMethodChain().get(0).getArguments().get(1); + // First statement is if, which contains a bare return + assertInstanceOf(MALExpressionModel.ClosureIfStatement.class, closure.getBody().get(0)); + final MALExpressionModel.ClosureIfStatement ifStmt = + (MALExpressionModel.ClosureIfStatement) closure.getBody().get(0); + assertInstanceOf(MALExpressionModel.ClosureReturnStatement.class, + ifStmt.getThenBranch().get(0)); + final MALExpressionModel.ClosureReturnStatement ret = + (MALExpressionModel.ClosureReturnStatement) ifStmt.getThenBranch().get(0); + // Bare return — value should be null + assertEquals(null, ret.getValue()); + } + + @Test + void parseStaticMethodCall() { + // ProcessRegistry.generateVirtualLocalProcess(tags.service, tags.instance) + final MALExpressionModel.Expr ast = MALScriptParser.parse( + "metric.tag({tags -> " + + "tags.pid = ProcessRegistry.generateVirtualLocalProcess(tags.service, tags.instance)" + + "})"); + assertInstanceOf(MetricExpr.class, ast); + final MetricExpr metric = (MetricExpr) ast; + final ClosureArgument closure = + (ClosureArgument) metric.getMethodChain().get(0).getArguments().get(0); + final MALExpressionModel.ClosureAssignment assign = + (MALExpressionModel.ClosureAssignment) closure.getBody().get(0); + // RHS should be a ClosureMethodChain with target "ProcessRegistry" + assertInstanceOf(MALExpressionModel.ClosureMethodChain.class, assign.getValue()); + final MALExpressionModel.ClosureMethodChain chain = + (MALExpressionModel.ClosureMethodChain) assign.getValue(); + assertEquals("ProcessRegistry", chain.getTarget()); + assertEquals(1, chain.getSegments().size()); + assertInstanceOf(MALExpressionModel.ClosureMethodCallSeg.class, + chain.getSegments().get(0)); + } + @Test void parseSyntaxErrorThrows() { assertThrows(IllegalArgumentException.class, diff --git a/oap-server/server-starter/pom.xml b/oap-server/server-starter/pom.xml index 35b5e7de81..109093dcdf 100644 --- a/oap-server/server-starter/pom.xml +++ b/oap-server/server-starter/pom.xml @@ -47,6 +47,13 @@ </dependency> <!-- OAL runtime core --> + <!-- Hierarchy rule compiler (SPI-loaded by server-core) --> + <dependency> + <groupId>org.apache.skywalking</groupId> + <artifactId>hierarchy</artifactId> + <version>${project.version}</version> + </dependency> + <!-- cluster module --> <dependency> <groupId>org.apache.skywalking</groupId> diff --git a/test/script-cases/script-runtime-with-groovy/mal-lal-v1-v2-checker/src/test/java/org/apache/skywalking/oap/server/checker/lal/LalComparisonTest.java b/test/script-cases/script-runtime-with-groovy/mal-lal-v1-v2-checker/src/test/java/org/apache/skywalking/oap/server/checker/lal/LalComparisonTest.java index e69c2185ac..7b70844715 100644 --- a/test/script-cases/script-runtime-with-groovy/mal-lal-v1-v2-checker/src/test/java/org/apache/skywalking/oap/server/checker/lal/LalComparisonTest.java +++ b/test/script-cases/script-runtime-with-groovy/mal-lal-v1-v2-checker/src/test/java/org/apache/skywalking/oap/server/checker/lal/LalComparisonTest.java @@ -19,7 +19,6 @@ package org.apache.skywalking.oap.server.checker.lal; import java.io.File; import java.lang.reflect.Field; -import java.lang.reflect.Method; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -62,7 +61,7 @@ import static org.mockito.Mockito.when; * <ul> * <li>Path A (v1): Groovy compilation + runtime execution via {@link DSL}</li> * <li>Path B (v2): ANTLR4 + Javassist compilation via {@link LALClassGenerator} - * + runtime execution via reflective {@code execute(Object, Object)} call</li> + * + runtime execution via {@code LalExpression.execute(FilterSpec, Binding)}</li> * </ul> * Both paths are fed the same mock LogData and the resulting Binding state * (service, layer, tags, abort/save flags) is compared. @@ -110,7 +109,8 @@ class LalComparisonTest { String v2Error = null; try { final LALClassGenerator generator = new LALClassGenerator(); - final Object v2Expr = generator.compile(dsl); + final org.apache.skywalking.oap.log.analyzer.dsl.LalExpression v2Expr = + generator.compile(dsl); final FilterSpec v2FilterSpec = new FilterSpec(manager, new LogAnalyzerModuleConfig()); disableSinkListenersOnSpec(v2FilterSpec); @@ -118,11 +118,7 @@ class LalComparisonTest { v2Binding = new Binding().log(testLog); v2FilterSpec.bind(v2Binding); - // Call execute(Object, Object) via reflection since the Javassist-generated - // class declares execute(Object, Object) rather than the typed signature - final Method executeMethod = v2Expr.getClass().getMethod("execute", - Object.class, Object.class); - executeMethod.invoke(v2Expr, v2FilterSpec, v2Binding); + v2Expr.execute(v2FilterSpec, v2Binding); } catch (Exception e) { final Throwable cause = e.getCause() != null ? e.getCause() : e; v2Error = cause.getClass().getSimpleName() + ": " + cause.getMessage(); @@ -286,14 +282,11 @@ class LalComparisonTest { return result; } - final File[] files = lalDir.toFile().listFiles(); - if (files == null) { - return result; - } - for (final File file : files) { - if (!file.getName().endsWith(".yaml") && !file.getName().endsWith(".yml")) { - continue; - } + // Scan top-level and subdirectories (oap-cases/, feature-cases/) + final List<File> yamlFiles = new ArrayList<>(); + collectYamlFiles(lalDir.toFile(), yamlFiles); + + for (final File file : yamlFiles) { final String content = Files.readString(file.toPath()); final Map<String, Object> config = yaml.load(content); if (config == null || !config.containsKey("rules")) { @@ -314,7 +307,8 @@ class LalComparisonTest { lalRules.add(new LalRule(name, dslStr)); } if (!lalRules.isEmpty()) { - result.put("lal/" + file.getName(), lalRules); + final String relative = lalDir.relativize(file.toPath()).toString(); + result.put("lal/" + relative, lalRules); } } return result; @@ -334,6 +328,22 @@ class LalComparisonTest { return null; } + private static void collectYamlFiles(final File dir, + final List<File> result) { + final File[] children = dir.listFiles(); + if (children == null) { + return; + } + for (final File child : children) { + if (child.isDirectory()) { + collectYamlFiles(child, result); + } else if (child.getName().endsWith(".yaml") + || child.getName().endsWith(".yml")) { + result.add(child); + } + } + } + private static class LalRule { final String name; final String dsl; diff --git a/test/script-cases/script-runtime-with-groovy/mal-lal-v1-v2-checker/src/test/java/org/apache/skywalking/oap/server/checker/mal/MalComparisonTest.java b/test/script-cases/script-runtime-with-groovy/mal-lal-v1-v2-checker/src/test/java/org/apache/skywalking/oap/server/checker/mal/MalComparisonTest.java index 6784e3852a..991dd0e317 100644 --- a/test/script-cases/script-runtime-with-groovy/mal-lal-v1-v2-checker/src/test/java/org/apache/skywalking/oap/server/checker/mal/MalComparisonTest.java +++ b/test/script-cases/script-runtime-with-groovy/mal-lal-v1-v2-checker/src/test/java/org/apache/skywalking/oap/server/checker/mal/MalComparisonTest.java @@ -21,10 +21,14 @@ import java.io.File; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import com.google.common.collect.ImmutableMap; import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.oap.meter.analyzer.compiler.MALClassGenerator; import org.apache.skywalking.oap.meter.analyzer.dsl.DSL; @@ -32,6 +36,10 @@ import org.apache.skywalking.oap.meter.analyzer.dsl.Expression; import org.apache.skywalking.oap.meter.analyzer.dsl.ExpressionMetadata; import org.apache.skywalking.oap.meter.analyzer.dsl.ExpressionParsingContext; import org.apache.skywalking.oap.meter.analyzer.dsl.MalExpression; +import org.apache.skywalking.oap.meter.analyzer.dsl.Result; +import org.apache.skywalking.oap.meter.analyzer.dsl.Sample; +import org.apache.skywalking.oap.meter.analyzer.dsl.SampleFamily; +import org.apache.skywalking.oap.meter.analyzer.dsl.SampleFamilyBuilder; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.TestFactory; import org.yaml.snakeyaml.Yaml; @@ -46,12 +54,21 @@ import static org.junit.jupiter.api.Assertions.fail; * <li>Path A (v1): Groovy compilation via upstream {@code DSL.parse()}</li> * <li>Path B (v2): ANTLR4 + Javassist compilation via {@link MALClassGenerator}</li> * </ul> - * Both paths run metadata extraction and compare the resulting metadata - * (samples, scope, downsampling, aggregation labels). + * Both paths run metadata extraction and runtime execution comparison. + * Metadata: compare samples, scope, downsampling, aggregation labels. + * Runtime: execute with mock SampleFamily data and compare output. */ @Slf4j class MalComparisonTest { + private static final Pattern TAG_EQUAL_PATTERN = + Pattern.compile("\\.tagEqual\\s*\\(\\s*'([^']+)'\\s*,\\s*'([^']+)'\\s*\\)"); + + private static final String[] HISTOGRAM_LE_VALUES = + {"50", "100", "250", "500", "1000"}; + + private long timestampCounter = System.currentTimeMillis(); + @TestFactory Collection<DynamicTest> malExpressionsMatch() throws Exception { final List<DynamicTest> tests = new ArrayList<>(); @@ -73,26 +90,28 @@ class MalComparisonTest { private void compareExpression(final String metricName, final String expression) throws Exception { // ---- V1: Groovy path ---- + Expression v1Expr = null; ExpressionParsingContext v1Ctx = null; try { - final Expression v1Expr = DSL.parse(metricName, expression); + v1Expr = DSL.parse(metricName, expression); v1Ctx = v1Expr.parse(); } catch (Exception e) { // V1 failed - skip comparison } // ---- V2: ANTLR4 + Javassist compilation ---- + MalExpression v2MalExpr = null; ExpressionMetadata v2Meta = null; String v2Error = null; try { final MALClassGenerator generator = new MALClassGenerator(); - final MalExpression malExpr = generator.compile(metricName, expression); - v2Meta = malExpr.metadata(); + v2MalExpr = generator.compile(metricName, expression); + v2Meta = v2MalExpr.metadata(); } catch (Exception e) { v2Error = e.getMessage(); } - // ---- Compare ---- + // ---- Compare metadata ---- if (v1Ctx == null && v2Meta == null) { // Both failed — consistent behavior return; @@ -103,6 +122,7 @@ class MalComparisonTest { } if (v2Meta == null) { fail(metricName + ": v2 compile failed but v1 succeeded — " + v2Error); + return; } // Both succeeded - compare metadata @@ -118,6 +138,210 @@ class MalComparisonTest { metricName + ": scopeLabels mismatch"); assertEquals(v1Ctx.getAggregationLabels(), v2Meta.getAggregationLabels(), metricName + ": aggregationLabels mismatch"); + + // ---- Runtime execution comparison ---- + compareExecution(metricName, expression, v1Expr, v2MalExpr, v2Meta); + } + + private void compareExecution(final String metricName, + final String expression, + final Expression v1Expr, + final MalExpression v2MalExpr, + final ExpressionMetadata v2Meta) { + final boolean hasIncrease = expression.contains(".increase(") + || expression.contains(".rate("); + + // For increase()/rate(), prime the CounterWindow with initial data + if (hasIncrease) { + try { + v1Expr.run(buildMockData(metricName, expression, v2Meta)); + } catch (Exception ignored) { + } + try { + final Map<String, SampleFamily> primeData = + buildMockData(metricName, expression, v2Meta); + for (final SampleFamily s : primeData.values()) { + if (s != SampleFamily.EMPTY) { + s.context.setMetricName(metricName); + } + } + v2MalExpr.run(primeData); + } catch (Exception ignored) { + } + } + + // Build fresh test data for actual comparison + final Map<String, SampleFamily> v1Data = + buildMockData(metricName, expression, v2Meta); + final Map<String, SampleFamily> v2Data = + buildMockData(metricName, expression, v2Meta); + + // V1 run + Result v1Result; + try { + v1Result = v1Expr.run(v1Data); + } catch (Exception e) { + // V1 runtime failed — skip comparison + return; + } + + // V2 run + SampleFamily v2Sf; + try { + for (final SampleFamily s : v2Data.values()) { + if (s != SampleFamily.EMPTY) { + s.context.setMetricName(metricName); + } + } + v2Sf = v2MalExpr.run(v2Data); + } catch (Exception e) { + if (v1Result.isSuccess()) { + fail(metricName + ": v2 runtime failed but v1 succeeded — " + + e.getClass().getSimpleName() + ": " + e.getMessage()); + } + return; + } + + // Compare results + final boolean v2Success = v2Sf != null && v2Sf != SampleFamily.EMPTY; + assertEquals(v1Result.isSuccess(), v2Success, + metricName + ": success mismatch (v1=" + v1Result.isSuccess() + + ", v2=" + v2Success + ")"); + + if (v1Result.isSuccess() && v2Success) { + compareSampleFamilies(metricName, v1Result.getData(), v2Sf); + } + } + + private Map<String, SampleFamily> buildMockData(final String metricName, + final String expression, + final ExpressionMetadata meta) { + final Map<String, SampleFamily> data = new HashMap<>(); + final long now = timestampCounter++; + + // Extract tagEqual constraints from the expression + final Map<String, String> tagEqualLabels = extractTagEqualLabels(expression); + + for (final String sampleName : meta.getSamples()) { + // Build labels from aggregation labels + final Map<String, String> labels = new HashMap<>(); + for (final String label : meta.getAggregationLabels()) { + labels.put(label, inferLabelValue(label, tagEqualLabels)); + } + // Also add tagEqual labels not in aggregation labels + labels.putAll(tagEqualLabels); + + if (meta.isHistogram()) { + data.put(sampleName, + buildHistogramSamples(sampleName, labels, now)); + } else { + final Sample sample = Sample.builder() + .name(sampleName) + .labels(ImmutableMap.copyOf(labels)) + .value(100.0) + .timestamp(now) + .build(); + data.put(sampleName, + SampleFamilyBuilder.newBuilder(sample).build()); + } + } + return data; + } + + private SampleFamily buildHistogramSamples(final String sampleName, + final Map<String, String> baseLabels, + final long timestamp) { + final List<Sample> samples = new ArrayList<>(); + double cumulativeValue = 0; + for (final String le : HISTOGRAM_LE_VALUES) { + cumulativeValue += 10.0; + final Map<String, String> labels = new HashMap<>(baseLabels); + labels.put("le", le); + samples.add(Sample.builder() + .name(sampleName) + .labels(ImmutableMap.copyOf(labels)) + .value(cumulativeValue) + .timestamp(timestamp) + .build()); + } + return SampleFamilyBuilder.newBuilder( + samples.toArray(new Sample[0])).build(); + } + + private static Map<String, String> extractTagEqualLabels(final String expression) { + final Map<String, String> labels = new HashMap<>(); + final Matcher matcher = TAG_EQUAL_PATTERN.matcher(expression); + while (matcher.find()) { + labels.put(matcher.group(1), matcher.group(2)); + } + return labels; + } + + private static String inferLabelValue(final String label, + final Map<String, String> tagEqualLabels) { + // If tagEqual specifies this label, use that value + if (tagEqualLabels.containsKey(label)) { + return tagEqualLabels.get(label); + } + switch (label) { + case "service": + return "test-service"; + case "instance": + case "service_instance_id": + return "test-instance"; + case "endpoint": + return "/test"; + case "host_name": + return "test-host"; + case "le": + return "100"; + case "job_name": + return "mysql-monitoring"; + case "cluster": + return "test-cluster"; + case "node": + case "node_id": + return "test-node"; + case "topic": + return "test-topic"; + case "queue": + return "test-queue"; + case "broker": + return "test-broker"; + default: + return "test-value"; + } + } + + private static void compareSampleFamilies(final String metricName, + final SampleFamily v1Sf, + final SampleFamily v2Sf) { + // Sort both sample arrays by labels for stable comparison + final Sample[] v1Sorted = sortSamples(v1Sf.samples); + final Sample[] v2Sorted = sortSamples(v2Sf.samples); + + assertEquals(v1Sorted.length, v2Sorted.length, + metricName + ": output sample count mismatch (v1=" + + v1Sorted.length + ", v2=" + v2Sorted.length + ")"); + + for (int i = 0; i < v1Sorted.length; i++) { + assertEquals(v1Sorted[i].getLabels(), v2Sorted[i].getLabels(), + metricName + ": output sample[" + i + "] labels mismatch"); + assertEquals(v1Sorted[i].getValue(), v2Sorted[i].getValue(), 0.001, + metricName + ": output sample[" + i + "] value mismatch" + + " (v1=" + v1Sorted[i].getValue() + + ", v2=" + v2Sorted[i].getValue() + ")"); + } + } + + private static Sample[] sortSamples(final Sample[] samples) { + final Sample[] sorted = Arrays.copyOf(samples, samples.length); + Arrays.sort(sorted, (a, b) -> { + final String aKey = a.getLabels().toString(); + final String bKey = b.getLabels().toString(); + return aKey.compareTo(bKey); + }); + return sorted; } @SuppressWarnings("unchecked") diff --git a/test/script-cases/scripts/lal/test-lal/feature-cases/execution-basic.input.data b/test/script-cases/scripts/lal/test-lal/feature-cases/execution-basic.input.data new file mode 100644 index 0000000000..de10ed6543 --- /dev/null +++ b/test/script-cases/scripts/lal/test-lal/feature-cases/execution-basic.input.data @@ -0,0 +1,157 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Mock input data and expected output for execution-basic.yaml rules. +# Format: YAML keyed by rule name. Each entry describes body-type, body, +# optional tags, and expected assertions after execution. + +json-parse-extract: + body-type: json + body: '{"service":"my-svc","instance":"inst-01","endpoint":"/api","layer":"GENERAL"}' + expect: + service: my-svc + instance: inst-01 + endpoint: /api + layer: GENERAL + save: true + abort: false + +tag-condition-true: + body-type: json + body: '{"service":"db-svc"}' + tags: + LOG_KIND: SLOW_SQL + expect: + service: db-svc + +tag-condition-false: + body-type: json + body: '{"service":"db-svc"}' + tags: + LOG_KIND: NORMAL + expect: + service: "" + +tag-assignment: + body-type: json + body: '{"env":"prod","region":"us-east"}' + expect: + tag.key1: prod + tag.key2: us-east + +safe-nav-missing: + body-type: json + body: '{"other":"val"}' + expect: + service: "" + abort: false + +safe-nav-present: + body-type: json + body: '{"data":{"name":"found"}}' + expect: + service: found + +if-else-if-error: + body-type: json + body: '{"level":"ERROR"}' + expect: + service: error-handler + +if-else-if-warn: + body-type: json + body: '{"level":"WARN"}' + expect: + service: warn-handler + +if-else-if-default: + body-type: json + body: '{"level":"DEBUG"}' + expect: + service: default-handler + +sink-enforcer: + body-type: json + body: '{}' + expect: + save: true + +sink-dropper: + body-type: json + body: '{}' + expect: + save: false + +sampler-rate-limit: + body-type: json + body: '{}' + expect: + save: true + +sampler-interpolated-id: + body-type: json + body: '{"code":"200"}' + expect: + save: true + +abort-stops-pipeline: + body-type: json + body: '{"service":"should-not-be-set"}' + expect: + abort: true + service: "" + +conditional-abort-true: + body-type: json + body: '{"skip":"true","service":"my-svc"}' + expect: + abort: true + service: "" + +conditional-abort-false: + body-type: json + body: '{"skip":"false","service":"my-svc"}' + expect: + abort: false + service: my-svc + +timestamp-extraction: + body-type: json + body: '{"time":"1609459200000"}' + expect: + timestamp: 1609459200000 + +text-parser-regexp: + body-type: text + body: "1609459200000 ERROR Something failed" + expect: + service: ERROR + +sampled-trace-basic: + body-type: json + body: '{"latency":150,"uri":"/test","reason":"slow","pid":"proc-a","dpid":"proc-b","dp":"client"}' + service: trace-svc + instance: trace-inst + trace-id: trace-basic-001 + timestamp: 1609459200000 + expect: + save: true + sampledTrace.latency: 150 + sampledTrace.uri: /test + sampledTrace.reason: SLOW + sampledTrace.processId: proc-a + sampledTrace.destProcessId: proc-b + sampledTrace.detectPoint: CLIENT + sampledTrace.componentId: 49 diff --git a/test/script-cases/scripts/lal/test-lal/feature-cases/execution-basic.yaml b/test/script-cases/scripts/lal/test-lal/feature-cases/execution-basic.yaml new file mode 100644 index 0000000000..71e3bb7d40 --- /dev/null +++ b/test/script-cases/scripts/lal/test-lal/feature-cases/execution-basic.yaml @@ -0,0 +1,272 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Feature-focused execution test rules for LAL v2 compiler. +# Each rule exercises a specific LAL language feature. +# Paired with execution-basic.input.data for mock input and expected output. +rules: + - name: json-parse-extract + layer: GENERAL + dsl: | + filter { + json {} + extractor { + service parsed.service as String + instance parsed.instance as String + endpoint parsed.endpoint as String + layer parsed.layer as String + } + sink {} + } + + - name: tag-condition-true + layer: GENERAL + dsl: | + filter { + json {} + if (tag("LOG_KIND") == "SLOW_SQL") { + extractor { + service parsed.service as String + } + sink {} + } + } + + - name: tag-condition-false + layer: GENERAL + dsl: | + filter { + json {} + if (tag("LOG_KIND") == "SLOW_SQL") { + extractor { + service parsed.service as String + } + sink {} + } + } + + - name: tag-assignment + layer: GENERAL + dsl: | + filter { + json {} + extractor { + tag key1: parsed.env as String, key2: parsed.region as String + } + sink {} + } + + - name: safe-nav-missing + layer: GENERAL + dsl: | + filter { + json {} + extractor { + service parsed?.missing?.deep as String + } + sink {} + } + + - name: safe-nav-present + layer: GENERAL + dsl: | + filter { + json {} + extractor { + service parsed?.data?.name as String + } + sink {} + } + + - name: if-else-if-error + layer: GENERAL + dsl: | + filter { + json {} + if (parsed.level == "ERROR") { + extractor { service "error-handler" as String } + sink {} + } else if (parsed.level == "WARN") { + extractor { service "warn-handler" as String } + sink {} + } else { + extractor { service "default-handler" as String } + sink {} + } + } + + - name: if-else-if-warn + layer: GENERAL + dsl: | + filter { + json {} + if (parsed.level == "ERROR") { + extractor { service "error-handler" as String } + sink {} + } else if (parsed.level == "WARN") { + extractor { service "warn-handler" as String } + sink {} + } else { + extractor { service "default-handler" as String } + sink {} + } + } + + - name: if-else-if-default + layer: GENERAL + dsl: | + filter { + json {} + if (parsed.level == "ERROR") { + extractor { service "error-handler" as String } + sink {} + } else if (parsed.level == "WARN") { + extractor { service "warn-handler" as String } + sink {} + } else { + extractor { service "default-handler" as String } + sink {} + } + } + + - name: sink-enforcer + layer: GENERAL + dsl: | + filter { + json {} + sink { + enforcer {} + } + } + + - name: sink-dropper + layer: GENERAL + dsl: | + filter { + json {} + sink { + dropper {} + } + } + + - name: sampler-rate-limit + layer: GENERAL + dsl: | + filter { + json {} + sink { + sampler { + rateLimit('test:svc') { + rpm 6000 + } + } + } + } + + - name: sampler-interpolated-id + layer: GENERAL + dsl: | + filter { + json {} + sink { + sampler { + rateLimit("${parsed.code}") { + rpm 6000 + } + } + } + } + + - name: abort-stops-pipeline + layer: GENERAL + dsl: | + filter { + json {} + abort {} + extractor { + service parsed.service as String + } + sink {} + } + + - name: conditional-abort-true + layer: GENERAL + dsl: | + filter { + json {} + if (parsed.skip == "true") { + abort {} + } + extractor { + service parsed.service as String + } + sink {} + } + + - name: conditional-abort-false + layer: GENERAL + dsl: | + filter { + json {} + if (parsed.skip == "true") { + abort {} + } + extractor { + service parsed.service as String + } + sink {} + } + + - name: timestamp-extraction + layer: GENERAL + dsl: | + filter { + json {} + extractor { + timestamp parsed.time as String + } + sink {} + } + + - name: text-parser-regexp + layer: GENERAL + dsl: | + filter { + text { + regexp $/(?<ts>\d+) (?<lvl>\w+) (?<msg>.*)/$ + } + extractor { + service parsed.lvl as String + } + sink {} + } + + - name: sampled-trace-basic + layer: MESH_DP + dsl: | + filter { + json {} + extractor { + sampledTrace { + latency parsed.latency as Long + uri parsed.uri as String + reason parsed.reason as String + processId parsed.pid as String + destProcessId parsed.dpid as String + detectPoint parsed.dp as String + componentId 49 + } + } + } diff --git a/test/script-cases/scripts/lal/test-lal/default.yaml b/test/script-cases/scripts/lal/test-lal/oap-cases/default.input.data similarity index 80% copy from test/script-cases/scripts/lal/test-lal/default.yaml copy to test/script-cases/scripts/lal/test-lal/oap-cases/default.input.data index 12317a95bf..6823207fb0 100644 --- a/test/script-cases/scripts/lal/test-lal/default.yaml +++ b/test/script-cases/scripts/lal/test-lal/oap-cases/default.input.data @@ -13,12 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -# The default LAL script to save all logs, behaving like the versions before 8.5.0. -rules: - - name: default - layer: GENERAL - dsl: | - filter { - sink { - } - } +# Mock input data for default.yaml rules. + +default: + body-type: json + body: '{}' + expect: + save: true + abort: false diff --git a/test/script-cases/scripts/lal/test-lal/default.yaml b/test/script-cases/scripts/lal/test-lal/oap-cases/default.yaml similarity index 100% copy from test/script-cases/scripts/lal/test-lal/default.yaml copy to test/script-cases/scripts/lal/test-lal/oap-cases/default.yaml diff --git a/test/script-cases/scripts/lal/test-lal/default.yaml b/test/script-cases/scripts/lal/test-lal/oap-cases/envoy-als.input.data similarity index 61% copy from test/script-cases/scripts/lal/test-lal/default.yaml copy to test/script-cases/scripts/lal/test-lal/oap-cases/envoy-als.input.data index 12317a95bf..948864414d 100644 --- a/test/script-cases/scripts/lal/test-lal/default.yaml +++ b/test/script-cases/scripts/lal/test-lal/oap-cases/envoy-als.input.data @@ -13,12 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -# The default LAL script to save all logs, behaving like the versions before 8.5.0. -rules: - - name: default - layer: GENERAL - dsl: | - filter { - sink { - } - } +# Mock input data for envoy-als.yaml rules. +# The envoy-als rule processes protobuf HTTPAccessLogEntry as extraLog, +# not JSON body. The proto-json is parsed via protobuf JsonFormat. + +envoy-als: + body-type: none + service: test-mesh-svc + extra-log: + proto-class: io.envoyproxy.envoy.data.accesslog.v3.HTTPAccessLogEntry + proto-json: '{"response":{"responseCode":500},"commonProperties":{"upstreamCluster":"outbound|80||backend.default.svc"}}' + expect: + tag.status.code: "500" diff --git a/test/script-cases/scripts/lal/test-lal/envoy-als.yaml b/test/script-cases/scripts/lal/test-lal/oap-cases/envoy-als.yaml similarity index 100% rename from test/script-cases/scripts/lal/test-lal/envoy-als.yaml rename to test/script-cases/scripts/lal/test-lal/oap-cases/envoy-als.yaml diff --git a/test/script-cases/scripts/lal/test-lal/oap-cases/k8s-service.input.data b/test/script-cases/scripts/lal/test-lal/oap-cases/k8s-service.input.data new file mode 100644 index 0000000000..11672b05f5 --- /dev/null +++ b/test/script-cases/scripts/lal/test-lal/oap-cases/k8s-service.input.data @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Mock input data for k8s-service.yaml rules. +# Tests the matching path: tag("LOG_KIND") == "NET_PROFILING_SAMPLED_TRACE", +# which enters the sampledTrace block and populates the SampledTraceBuilder. +# Uses HTTPS component (parsed.component == "http" && parsed.ssl == true → componentId 129). + +network-profiling-slow-trace: + body-type: json + body: '{"latency":350,"uri":"/k8s/endpoint","reason":"status_5xx","client_process":{"process_id":"k8s-client-proc","local":true,"address":"10.1.0.1:80"},"server_process":{"process_id":"k8s-server-proc","local":false,"address":"10.1.0.2:443"},"detect_point":"server","component":"http","ssl":true}' + service: k8s-test-svc + instance: k8s-test-instance + trace-id: trace-k8s-002 + timestamp: 1609459300000 + tags: + LOG_KIND: NET_PROFILING_SAMPLED_TRACE + expect: + save: true + abort: false + sampledTrace.latency: 350 + sampledTrace.uri: /k8s/endpoint + sampledTrace.reason: STATUS_5XX + sampledTrace.processId: k8s-client-proc + sampledTrace.destProcessId: k8s-server-proc + sampledTrace.detectPoint: SERVER + sampledTrace.componentId: 129 diff --git a/test/script-cases/scripts/lal/test-lal/k8s-service.yaml b/test/script-cases/scripts/lal/test-lal/oap-cases/k8s-service.yaml similarity index 100% rename from test/script-cases/scripts/lal/test-lal/k8s-service.yaml rename to test/script-cases/scripts/lal/test-lal/oap-cases/k8s-service.yaml diff --git a/test/script-cases/scripts/lal/test-lal/oap-cases/mesh-dp.input.data b/test/script-cases/scripts/lal/test-lal/oap-cases/mesh-dp.input.data new file mode 100644 index 0000000000..b60831681a --- /dev/null +++ b/test/script-cases/scripts/lal/test-lal/oap-cases/mesh-dp.input.data @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Mock input data for mesh-dp.yaml rules. +# Tests the matching path: tag("LOG_KIND") == "NET_PROFILING_SAMPLED_TRACE", +# which enters the sampledTrace block and populates the SampledTraceBuilder. +# We provide process_id directly to avoid K8s/ProcessRegistry dependency. + +network-profiling-slow-trace: + body-type: json + body: '{"latency":200,"uri":"/mesh/api","reason":"slow","client_process":{"process_id":"client-proc-1","local":false,"address":"10.0.0.1:8080"},"server_process":{"process_id":"server-proc-2","local":true,"address":"10.0.0.2:9090"},"detect_point":"client","component":"http","ssl":false}' + service: mesh-test-svc + instance: mesh-test-instance + trace-id: trace-mesh-001 + timestamp: 1609459200000 + tags: + LOG_KIND: NET_PROFILING_SAMPLED_TRACE + expect: + save: true + abort: false + sampledTrace.latency: 200 + sampledTrace.uri: /mesh/api + sampledTrace.reason: SLOW + sampledTrace.processId: client-proc-1 + sampledTrace.destProcessId: server-proc-2 + sampledTrace.detectPoint: CLIENT + sampledTrace.componentId: 49 diff --git a/test/script-cases/scripts/lal/test-lal/mesh-dp.yaml b/test/script-cases/scripts/lal/test-lal/oap-cases/mesh-dp.yaml similarity index 100% rename from test/script-cases/scripts/lal/test-lal/mesh-dp.yaml rename to test/script-cases/scripts/lal/test-lal/oap-cases/mesh-dp.yaml diff --git a/test/script-cases/scripts/lal/test-lal/default.yaml b/test/script-cases/scripts/lal/test-lal/oap-cases/mysql-slowsql.input.data similarity index 71% copy from test/script-cases/scripts/lal/test-lal/default.yaml copy to test/script-cases/scripts/lal/test-lal/oap-cases/mysql-slowsql.input.data index 12317a95bf..49d3f2ec27 100644 --- a/test/script-cases/scripts/lal/test-lal/default.yaml +++ b/test/script-cases/scripts/lal/test-lal/oap-cases/mysql-slowsql.input.data @@ -13,12 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -# The default LAL script to save all logs, behaving like the versions before 8.5.0. -rules: - - name: default - layer: GENERAL - dsl: | - filter { - sink { - } - } +# Mock input data for mysql-slowsql.yaml rules. + +mysql-slowsql: + body-type: json + body: '{"layer":"MYSQL","service":"db-svc","time":"1609459200000","id":"slow-1","statement":"SELECT 1","query_time":500}' + tags: + LOG_KIND: SLOW_SQL + expect: + service: db-svc + layer: MYSQL + timestamp: 1609459200000 diff --git a/test/script-cases/scripts/lal/test-lal/mysql-slowsql.yaml b/test/script-cases/scripts/lal/test-lal/oap-cases/mysql-slowsql.yaml similarity index 100% rename from test/script-cases/scripts/lal/test-lal/mysql-slowsql.yaml rename to test/script-cases/scripts/lal/test-lal/oap-cases/mysql-slowsql.yaml diff --git a/test/script-cases/scripts/lal/test-lal/default.yaml b/test/script-cases/scripts/lal/test-lal/oap-cases/nginx.input.data similarity index 76% copy from test/script-cases/scripts/lal/test-lal/default.yaml copy to test/script-cases/scripts/lal/test-lal/oap-cases/nginx.input.data index 12317a95bf..21ea01f7c1 100644 --- a/test/script-cases/scripts/lal/test-lal/default.yaml +++ b/test/script-cases/scripts/lal/test-lal/oap-cases/nginx.input.data @@ -13,12 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -# The default LAL script to save all logs, behaving like the versions before 8.5.0. -rules: - - name: default - layer: GENERAL - dsl: | - filter { - sink { - } - } +# Mock input data for nginx.yaml rules. + +nginx-access-log: + body-type: text + body: '10.0.0.1 - - [01/Jan/2021:00:00:00 +0000] "GET /api HTTP/1.1" 200 1234' + tags: + LOG_KIND: NGINX_ACCESS_LOG + expect: + tag.http.status_code: "200" diff --git a/test/script-cases/scripts/lal/test-lal/nginx.yaml b/test/script-cases/scripts/lal/test-lal/oap-cases/nginx.yaml similarity index 100% rename from test/script-cases/scripts/lal/test-lal/nginx.yaml rename to test/script-cases/scripts/lal/test-lal/oap-cases/nginx.yaml diff --git a/test/script-cases/scripts/lal/test-lal/default.yaml b/test/script-cases/scripts/lal/test-lal/oap-cases/pgsql-slowsql.input.data similarity index 70% copy from test/script-cases/scripts/lal/test-lal/default.yaml copy to test/script-cases/scripts/lal/test-lal/oap-cases/pgsql-slowsql.input.data index 12317a95bf..ffdae9b64b 100644 --- a/test/script-cases/scripts/lal/test-lal/default.yaml +++ b/test/script-cases/scripts/lal/test-lal/oap-cases/pgsql-slowsql.input.data @@ -13,12 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -# The default LAL script to save all logs, behaving like the versions before 8.5.0. -rules: - - name: default - layer: GENERAL - dsl: | - filter { - sink { - } - } +# Mock input data for pgsql-slowsql.yaml rules. + +pgsql-slowsql: + body-type: json + body: '{"layer":"POSTGRESQL","service":"pg-svc","time":"1609459200000","id":"slow-pg-1","statement":"SELECT 1","query_time":300}' + tags: + LOG_KIND: SLOW_SQL + expect: + service: pg-svc + layer: POSTGRESQL + timestamp: 1609459200000 diff --git a/test/script-cases/scripts/lal/test-lal/pgsql-slowsql.yaml b/test/script-cases/scripts/lal/test-lal/oap-cases/pgsql-slowsql.yaml similarity index 100% rename from test/script-cases/scripts/lal/test-lal/pgsql-slowsql.yaml rename to test/script-cases/scripts/lal/test-lal/oap-cases/pgsql-slowsql.yaml diff --git a/test/script-cases/scripts/lal/test-lal/default.yaml b/test/script-cases/scripts/lal/test-lal/oap-cases/redis-slowsql.input.data similarity index 70% rename from test/script-cases/scripts/lal/test-lal/default.yaml rename to test/script-cases/scripts/lal/test-lal/oap-cases/redis-slowsql.input.data index 12317a95bf..95dc05b494 100644 --- a/test/script-cases/scripts/lal/test-lal/default.yaml +++ b/test/script-cases/scripts/lal/test-lal/oap-cases/redis-slowsql.input.data @@ -13,12 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -# The default LAL script to save all logs, behaving like the versions before 8.5.0. -rules: - - name: default - layer: GENERAL - dsl: | - filter { - sink { - } - } +# Mock input data for redis-slowsql.yaml rules. + +redis-slowsql: + body-type: json + body: '{"layer":"REDIS","service":"redis-svc","time":"1609459200000","id":"slow-redis-1","statement":"GET key","query_time":200}' + tags: + LOG_KIND: SLOW_SQL + expect: + service: redis-svc + layer: REDIS + timestamp: 1609459200000 diff --git a/test/script-cases/scripts/lal/test-lal/redis-slowsql.yaml b/test/script-cases/scripts/lal/test-lal/oap-cases/redis-slowsql.yaml similarity index 100% rename from test/script-cases/scripts/lal/test-lal/redis-slowsql.yaml rename to test/script-cases/scripts/lal/test-lal/oap-cases/redis-slowsql.yaml
