This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new 0f8c67822d Support extract timestamp from patterned datetime string in
LAL. (#11489)
0f8c67822d is described below
commit 0f8c67822d9e07cc446e66579c57cfcb2e035297
Author: weixiang1862 <652048...@qq.com>
AuthorDate: Fri Nov 3 14:42:04 2023 +0800
Support extract timestamp from patterned datetime string in LAL. (#11489)
---
docs/en/changes/changes.md | 1 +
docs/en/concepts-and-designs/lal.md | 21 ++++++++++++++-
.../analyzer/dsl/spec/extractor/ExtractorSpec.java | 25 ++++++++++++++++--
.../skywalking/oap/log/analyzer/dsl/DSLTest.java | 11 ++++++++
.../query/graphql/resolver/LogTestQueryTest.java | 30 ++++++++++++++++++++++
test/e2e-v2/cases/vm/zabbix/docker-compose.yml | 2 +-
6 files changed, 86 insertions(+), 4 deletions(-)
diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
index e464311ec2..893f5f4ae8 100644
--- a/docs/en/changes/changes.md
+++ b/docs/en/changes/changes.md
@@ -35,6 +35,7 @@
* BanyanDBStorageClient: Add `define(Property property, PropertyStore.Strategy
strategy)` API.
* Support GraalVM native-image (Experimental).
* Correct the file format and fix typos in the filenames for monitoring
Kafka's e2e tests.
+* Support extract timestamp from patterned datetime string in LAL.
#### UI
diff --git a/docs/en/concepts-and-designs/lal.md
b/docs/en/concepts-and-designs/lal.md
index f81d96f125..caa934a923 100644
--- a/docs/en/concepts-and-designs/lal.md
+++ b/docs/en/concepts-and-designs/lal.md
@@ -185,7 +185,26 @@ dropped) and is used to associate with traces / metrics.
`timestamp` extracts the timestamp from the `parsed` result, and set it into
the `LogData`, which will be persisted (if
not dropped) and is used to associate with traces / metrics.
-The unit of `timestamp` is millisecond.
+The parameter of `timestamp` can be a millisecond:
+```groovy
+filter {
+ // ... parser
+
+ extractor {
+ timestamp parsed.time as String
+ }
+}
+```
+or a datetime string with a specified pattern:
+```groovy
+filter {
+ // ... parser
+
+ extractor {
+ timestamp parsed.time as String, "yyyy-MM-dd HH:mm:ss"
+ }
+}
+```
- `layer`
diff --git
a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/extractor/ExtractorSpec.java
b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/extractor/ExtractorSpec.java
index 9359fff070..2a52cd64e1 100644
---
a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/extractor/ExtractorSpec.java
+++
b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/extractor/ExtractorSpec.java
@@ -22,6 +22,8 @@ import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableMap;
import groovy.lang.Closure;
import groovy.lang.DelegatesTo;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
import java.util.Collection;
import java.util.List;
import java.util.Map;
@@ -57,6 +59,7 @@ import
org.apache.skywalking.oap.server.core.source.SourceReceiver;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
import org.apache.skywalking.oap.server.library.module.ModuleStartException;
import org.apache.skywalking.oap.server.library.util.CollectionUtils;
+import org.apache.skywalking.oap.server.library.util.StringUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -203,11 +206,29 @@ public class ExtractorSpec extends AbstractSpec {
@SuppressWarnings("unused")
public void timestamp(final String timestamp) {
+ timestamp(timestamp, null);
+ }
+
+ @SuppressWarnings("unused")
+ public void timestamp(final String timestamp, final String formatPattern) {
if (BINDING.get().shouldAbort()) {
return;
}
- if (nonNull(timestamp) && StringUtils.isNumeric(timestamp)) {
- BINDING.get().log().setTimestamp(Long.parseLong(timestamp));
+ if (StringUtil.isEmpty(timestamp)) {
+ return;
+ }
+
+ if (StringUtil.isEmpty(formatPattern)) {
+ if (StringUtils.isNumeric(timestamp)) {
+ BINDING.get().log().setTimestamp(Long.parseLong(timestamp));
+ }
+ } else {
+ SimpleDateFormat format = new SimpleDateFormat(formatPattern);
+ try {
+
BINDING.get().log().setTimestamp(format.parse(timestamp).getTime());
+ } catch (ParseException e) {
+ // ignore
+ }
}
}
diff --git
a/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/dsl/DSLTest.java
b/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/dsl/DSLTest.java
index 333f164cf6..163595a3ff 100644
---
a/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/dsl/DSLTest.java
+++
b/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/dsl/DSLTest.java
@@ -176,6 +176,17 @@ public class DSLTest {
" }\n" +
" }\n" +
" }"
+ },
+ new String[] {
+ "extractor-patterned-timestamp",
+ "filter {\n" +
+ " extractor {\n" +
+ " service \"test\"\n" +
+ " instance \"test\"\n" +
+ " endpoint \"test\"\n" +
+ " timestamp \"2023-11-01 22:10:10\", \"yyyy-MM-dd
HH:mm:ss\"\n" +
+ " }\n" +
+ "}",
}
);
}
diff --git
a/oap-server/server-query-plugin/query-graphql-plugin/src/test/java/org/apache/skywalking/oap/query/graphql/resolver/LogTestQueryTest.java
b/oap-server/server-query-plugin/query-graphql-plugin/src/test/java/org/apache/skywalking/oap/query/graphql/resolver/LogTestQueryTest.java
index b60894dee4..035e6568e4 100644
---
a/oap-server/server-query-plugin/query-graphql-plugin/src/test/java/org/apache/skywalking/oap/query/graphql/resolver/LogTestQueryTest.java
+++
b/oap-server/server-query-plugin/query-graphql-plugin/src/test/java/org/apache/skywalking/oap/query/graphql/resolver/LogTestQueryTest.java
@@ -135,4 +135,34 @@ public class LogTestQueryTest {
assertEquals(1, response.getMetrics().iterator().next().getValue());
assertEquals(12312313,
response.getMetrics().iterator().next().getTimestamp());
}
+
+ @Test
+ public void testExtractPatternedTimestamp() throws Exception {
+ when(config.isEnableLogTestTool()).thenReturn(true);
+ final LogTestQuery query = new LogTestQuery(moduleManager, config);
+ final LogTestRequest request = new LogTestRequest();
+ request.setLog("" +
+ "{" +
+ " body: {" +
+ " json: {" +
+ " json: '{\"request\": \"GET /l
HTTP/1.1\",\"time\": \"2023-11-02T12:39:36+00:00\",\"status\":
\"404\",\"request_time\":\"0.000\"}'" +
+ " }" +
+ " }," +
+ " type: JSON," +
+ " timestamp: 12312313," +
+ " service: 'test'" +
+ "}");
+ request.setDsl("" +
+ "filter {\n" +
+ " json {" +
+ " }\n" +
+ " extractor {\n" +
+ " timestamp parsed.time as String,
\"yyyy-MM-dd'T'HH:mm:ssXXX\"\n" +
+ " }\n" +
+ " sink {\n" +
+ " }\n" +
+ "}");
+ final LogTestResponse response = query.test(request);
+ assertEquals(1698928776000L, response.getLog().getTimestamp());
+ }
}
diff --git a/test/e2e-v2/cases/vm/zabbix/docker-compose.yml
b/test/e2e-v2/cases/vm/zabbix/docker-compose.yml
index 6a461ef422..b67b9349a4 100644
--- a/test/e2e-v2/cases/vm/zabbix/docker-compose.yml
+++ b/test/e2e-v2/cases/vm/zabbix/docker-compose.yml
@@ -26,7 +26,7 @@ services:
- 12800
zabbix-client:
- image: zabbix/zabbix-agent:latest
+ image: zabbix/zabbix-agent:alpine-latest
networks:
- e2e
volumes:
