This is an automated email from the ASF dual-hosted git repository. jbonofre pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/karaf-decanter.git
The following commit(s) were added to refs/heads/master by this push: new 97c0328 [KARAF-6990] Add loki appender new 4d0a8a3 Merge pull request #240 from jbonofre/KARAF-6990 97c0328 is described below commit 97c0328609e12bf8c24bb6cedec1063e818c969f Author: jbonofre <jbono...@apache.org> AuthorDate: Sat Feb 6 06:25:54 2021 +0100 [KARAF-6990] Add loki appender --- appender/loki/NOTICE | 57 ++++++++++ appender/loki/pom.xml | 91 ++++++++++++++++ .../org.apache.karaf.decanter.appender.loki.cfg | 35 +++++++ .../karaf/decanter/appender/loki/LokiAppender.java | 116 +++++++++++++++++++++ appender/pom.xml | 1 + assembly/src/main/feature/feature.xml | 8 +- manual/src/main/asciidoc/user-guide/appenders.adoc | 38 +++++++ 7 files changed, 345 insertions(+), 1 deletion(-) diff --git a/appender/loki/NOTICE b/appender/loki/NOTICE new file mode 100644 index 0000000..4e4af9e --- /dev/null +++ b/appender/loki/NOTICE @@ -0,0 +1,57 @@ +Apache Karaf Decanter +Copyright 2015-2019 The Apache Software Foundation + +I. Included Software + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). +Licensed under the Apache License 2.0. + +This product includes software developed at +Elastic (https://www.elastic.co/). +Licensed under the Apache License 2.0. + +This product includes software developed at +OrientDB (http://orientdb.com). +Licensed under the Apache License 2.0. + +II. Used Software + +This product uses software developed at +The OSGi Alliance (http://www.osgi.org/). +Copyright (c) OSGi Alliance (2000, 2010). +Licensed under the Apache License 2.0. + +This product uses software developed at +OPS4J (http://www.ops4j.org/). +Licensed under the Apache License 2.0. + +This product uses software developed at +SLF4J (http://www.slf4j.org/). +Licensed under the MIT License. + +This product uses software developed at +JUnit (http://www.junit.org/). +Licensed under the Eclipse Public License 1.0. + +This product uses software developed at +Redis (http://www.redis.io). +Licensed under the BSD license. + +This product uses software developed at +Dropwizard (http://www.dropwizard.io). +Licensed under the Apache License 2.0. + +This product uses software developed at +searchbox.io (https://github.com/searchbox-io) +Licensed under the Apache License 2.0. + +This product uses software developed at +MongoDB (https://www.mongodb.com/) +Licensed under the Apache License 2.0. + +III. License Summary +- Apache License 2.0 +- MIT License +- Eclipse Public License 1.0 +- BSD License diff --git a/appender/loki/pom.xml b/appender/loki/pom.xml new file mode 100644 index 0000000..d603dc8 --- /dev/null +++ b/appender/loki/pom.xml @@ -0,0 +1,91 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + + <!-- + + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.karaf.decanter</groupId> + <artifactId>appender</artifactId> + <version>2.7.0-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + + <groupId>org.apache.karaf.decanter.appender</groupId> + <artifactId>org.apache.karaf.decanter.appender.loki</artifactId> + <packaging>bundle</packaging> + <name>Apache Karaf :: Decanter :: Appender :: Loki</name> + + <dependencies> + <dependency> + <groupId>org.apache.karaf.decanter</groupId> + <artifactId>org.apache.karaf.decanter.api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.karaf.decanter.appender</groupId> + <artifactId>org.apache.karaf.decanter.appender.utils</artifactId> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.felix</groupId> + <artifactId>maven-bundle-plugin</artifactId> + <inherited>true</inherited> + <extensions>true</extensions> + <configuration> + <obrRepository>NONE</obrRepository> + <instructions> + <Bundle-SymbolicName>${project.artifactId}</Bundle-SymbolicName> + <Export-Package>!*</Export-Package> + <Import-Package>*</Import-Package> + <Private-Package> + org.apache.karaf.decanter.appender.loki, + org.apache.karaf.decanter.appender.utils + </Private-Package> + <_dsannotations>*</_dsannotations> + </instructions> + </configuration> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>attach-artifact</goal> + </goals> + <configuration> + <artifacts> + <artifact> + <file>src/main/cfg/org.apache.karaf.decanter.appender.loki.cfg</file> + <type>cfg</type> + </artifact> + </artifacts> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + +</project> \ No newline at end of file diff --git a/appender/loki/src/main/cfg/org.apache.karaf.decanter.appender.loki.cfg b/appender/loki/src/main/cfg/org.apache.karaf.decanter.appender.loki.cfg new file mode 100644 index 0000000..1e21de7 --- /dev/null +++ b/appender/loki/src/main/cfg/org.apache.karaf.decanter.appender.loki.cfg @@ -0,0 +1,35 @@ +################################################################################ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +###################################### +# Decanter Loki Appender Configuration +###################################### + +# Loki push API location +#loki.url=http://localhost:3100/loki/api/v1/push + +# Loki tenant +#loki.tenant=my-tenant + +# Loki basic authentication +#loki.username= +#loki.password= + +# Marshaller +#marshaller.target=(dataFormat=raw) \ No newline at end of file diff --git a/appender/loki/src/main/java/org/apache/karaf/decanter/appender/loki/LokiAppender.java b/appender/loki/src/main/java/org/apache/karaf/decanter/appender/loki/LokiAppender.java new file mode 100644 index 0000000..58c1bba --- /dev/null +++ b/appender/loki/src/main/java/org/apache/karaf/decanter/appender/loki/LokiAppender.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.karaf.decanter.appender.loki; + +import org.apache.karaf.decanter.api.marshaller.Marshaller; +import org.apache.karaf.decanter.appender.utils.EventFilter; +import org.osgi.service.component.ComponentContext; +import org.osgi.service.component.annotations.Activate; +import org.osgi.service.component.annotations.Component; +import org.osgi.service.component.annotations.Reference; +import org.osgi.service.component.annotations.ReferenceCardinality; +import org.osgi.service.event.Event; +import org.osgi.service.event.EventConstants; +import org.osgi.service.event.EventHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedWriter; +import java.io.OutputStreamWriter; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Dictionary; + +@Component( + name = "org.apache.karaf.decanter.appender.loki", + immediate = true, + property = EventConstants.EVENT_TOPIC + "=decanter/collect/*" +) +public class LokiAppender implements EventHandler { + + private final static Logger LOGGER = LoggerFactory.getLogger(LokiAppender.class); + + @Reference(cardinality = ReferenceCardinality.OPTIONAL) + public Marshaller marshaller; + + private String url; + private String tenant = null; + private String username = null; + private String password = null; + private Dictionary<String, Object> config; + + @Activate + public void activate(ComponentContext componentContext) { + activate(componentContext.getProperties()); + } + + public void activate(Dictionary<String, Object> config) { + this.config = config; + url = (config.get("loki.url") != null) ? (String) config.get("loki.url") : "http://localhost:3100/loki/api/v1/push"; + tenant = (config.get("loki.tenant") != null) ? (String) config.get("loki.tenant") : null; + username = (config.get("loki.username") != null) ? (String) config.get("loki.username") : null; + password = (config.get("loki.password") != null) ? (String) config.get("loki.password") : null; + } + + @Override + public void handleEvent(Event event) { + if (EventFilter.match(event, config)) { + String log; + if (marshaller != null) { + log = marshaller.marshal(event); + } else { + StringBuilder builder = new StringBuilder(); + for (String innerKey : event.getPropertyNames()) { + builder.append(innerKey).append(":").append(toString(event.getProperty(innerKey))).append(" | "); + } + log = builder.toString(); + } + try { + HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); + connection.setDoOutput(true); + connection.setRequestMethod("POST"); + connection.setRequestProperty("Content-Type", "application/json"); + if (tenant != null) { + connection.setRequestProperty("X-Scope-OrgId", tenant); + } + if (username != null) { + String authentication = username + ":" + password; + byte[] encodedAuthentication = Base64.getEncoder().encode(authentication.getBytes(StandardCharsets.UTF_8)); + String authenticationHeader = "Basic " + new String(encodedAuthentication); + connection.setRequestProperty("Authorization", authenticationHeader); + } + String jsonPush = "{\"streams\": [{ \"stream\": { \"job\": \"decanter\" }, \"values\": [ [ \"" + System.currentTimeMillis() * 1000L * 1000L + "\", \"" + log + "\" ] ] }]}"; + try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(connection.getOutputStream()))) { + writer.write(jsonPush); + writer.flush(); + } + if (connection.getResponseCode() != 204) { + LOGGER.warn("Can't push to Loki ({}): {}", connection.getResponseCode(), connection.getResponseMessage()); + } + } catch (Exception e) { + LOGGER.warn("Error occurred while pushing to Loki", e); + } + } + } + + private Object toString(Object value) { + return value == null ? null : value.toString(); + } + +} diff --git a/appender/pom.xml b/appender/pom.xml index 1d43b67..8e7b5bc 100644 --- a/appender/pom.xml +++ b/appender/pom.xml @@ -45,6 +45,7 @@ <module>jms</module> <module>kafka</module> <module>log</module> + <module>loki</module> <module>orientdb</module> <module>mongodb</module> <module>mqtt</module> diff --git a/assembly/src/main/feature/feature.xml b/assembly/src/main/feature/feature.xml index 8302043..3ed6305 100644 --- a/assembly/src/main/feature/feature.xml +++ b/assembly/src/main/feature/feature.xml @@ -468,13 +468,19 @@ org.apache.felix.eventadmin.IgnoreTimeout=org.apache.karaf.decanter. <bundle>mvn:org.apache.karaf.decanter.appender/org.apache.karaf.decanter.appender.websocket-servlet/${project.version}</bundle> </feature> - <feature name="decanter-appender-prometheus" version="${project.Version}" description="Karaf Decanter Prometheus Servlet Appender"> + <feature name="decanter-appender-prometheus" version="${project.version}" description="Karaf Decanter Prometheus Servlet Appender"> <feature>decanter-common</feature> <feature>http</feature> <configfile finalname="/etc/org.apache.karaf.decanter.appender.prometheus.cfg">mvn:org.apache.karaf.decanter.appender/org.apache.karaf.decanter.appender.prometheus/${project.version}/cfg</configfile> <bundle>mvn:org.apache.karaf.decanter.appender/org.apache.karaf.decanter.appender.prometheus/${project.version}</bundle> </feature> + <feature name="decanter-appender-loki" version="${project.version}" description="Karaf Decanter Loki appender"> + <feature>decanter-common</feature> + <configfile finalname="/etc/org.apache.karaf.decanter.appender.loki.cfg">mvn:org.apache.karaf.decanter.appender/org.apache.karaf.decanter.appender.loki/${project.version}/cfg</configfile> + <bundle>mvn:org.apache.karaf.decanter.appender/org.apache.karaf.decanter.appender.loki/${project.version}</bundle> + </feature> + <feature name="decanter-processor-passthrough" version="${project.version}" description="Karaf Decanter Pass Through Processor"> <feature>decanter-common</feature> <bundle>mvn:org.apache.karaf.decanter.processor/org.apache.karaf.decanter.processor.passthrough/${project.version}</bundle> diff --git a/manual/src/main/asciidoc/user-guide/appenders.adoc b/manual/src/main/asciidoc/user-guide/appenders.adoc index 739962b..97e3832 100644 --- a/manual/src/main/asciidoc/user-guide/appenders.adoc +++ b/manual/src/main/asciidoc/user-guide/appenders.adoc @@ -321,6 +321,44 @@ This file allows you to define how the messages are sent to the Kafka broker: It's also possible to enable SSL security (with Kafka 0.9.x) using the SSL properties. +==== Loki + +Loki (https://grafana.com/oss/loki/) is a log aggregation system. The Decanter Loki appender is able to push collected data +to Loki via the Push API. + +The Decanter Loki appender converts any kind of collected data (coming from the dispatcher) as a log string that can be stored in Loki. + +The `decanter-appender-loki` feature installs the Loki appender: + +---- +karaf@root()> feature:install decanter-appender-loki +---- + +This feature also adds `etc/org.apache.karaf.decanter.appender.loki.cfg` configuration file: + +---- +###################################### +# Decanter Loki Appender Configuration +###################################### + +# Loki push API location +#loki.url=http://localhost:3100/loki/api/v1/push + +# Loki tenant +#loki.tenant=my-tenant + +# Loki basic authentication +#loki.username= +#loki.password= + +# Marshaller +#marshaller.target=(dataFormat=raw) +---- + +* `loki.url` is the location of the Loki push API +* `loki.tenant` is optional and define the tenant used to push data +* `loki.username` and `loki.password` are used for basic authentication + ==== Redis The Decanter Redis appender sends the data (collected by the collectors) to a Redis broker.