This is an automated email from the ASF dual-hosted git repository.
markusthoemmes pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git
The following commit(s) were added to refs/heads/master by this push:
new 6f1a445 Support tags in metrics for finer granularity. (#3343)
6f1a445 is described below
commit 6f1a445f22f827bd91f75b01bc82988054febbc8
Author: Seong-hyun, Oh <[email protected]>
AuthorDate: Thu Mar 1 23:37:35 2018 +0900
Support tags in metrics for finer granularity. (#3343)
---
ansible/group_vars/all | 2 +
ansible/roles/controller/tasks/deploy.yml | 1 +
ansible/roles/invoker/tasks/deploy.yml | 1 +
.../src/main/scala/whisk/common/Logging.scala | 67 ++++++++++++++++------
.../main/scala/whisk/common/TransactionId.scala | 7 +--
.../main/scala/whisk/http/BasicHttpService.scala | 8 ++-
docs/metrics.md | 7 +++
.../docker/test/DockerClientTests.scala | 8 +--
.../docker/test/RuncClientTests.scala | 4 +-
9 files changed, 76 insertions(+), 29 deletions(-)
diff --git a/ansible/group_vars/all b/ansible/group_vars/all
index ce20543..a15a37a 100644
--- a/ansible/group_vars/all
+++ b/ansible/group_vars/all
@@ -301,5 +301,7 @@ metrics:
enabled: "{{ metrics_log | default(true) }}"
kamon:
enabled: "{{ metrics_kamon | default(false) }}"
+ tags: "{{ metrics_kamon_tags | default(false) }}"
host: "{{ metrics_kamon_statsd_host | default('') }}"
port: "{{ metrics_kamon_statsd_port | default('8125') }}"
+
diff --git a/ansible/roles/controller/tasks/deploy.yml
b/ansible/roles/controller/tasks/deploy.yml
index 52dbc80..e466d8b 100644
--- a/ansible/roles/controller/tasks/deploy.yml
+++ b/ansible/roles/controller/tasks/deploy.yml
@@ -152,6 +152,7 @@
"AKKA_CLUSTER_SEED_NODES": "{{seed_nodes_list | join(' ') }}"
"METRICS_KAMON": "{{ metrics.kamon.enabled }}"
+ "METRICS_KAMON_TAGS": "{{ metrics.kamon.tags }}"
"METRICS_LOG": "{{ metrics.log.enabled }}"
"CONFIG_whisk_loadbalancer_invokerBusyThreshold": "{{
invoker.busyThreshold }}"
diff --git a/ansible/roles/invoker/tasks/deploy.yml
b/ansible/roles/invoker/tasks/deploy.yml
index 0f3bdc6..8fc7a84 100644
--- a/ansible/roles/invoker/tasks/deploy.yml
+++ b/ansible/roles/invoker/tasks/deploy.yml
@@ -194,6 +194,7 @@
-e INVOKER_NAME='{{ groups['invokers'].index(inventory_hostname) }}'
-e WHISK_LOGS_DIR='{{ whisk_logs_dir }}'
-e METRICS_KAMON='{{ metrics.kamon.enabled }}'
+ -e METRICS_KAMON_TAGS='{{ metrics.kamon.tags }}'
-e METRICS_LOG='{{ metrics.log.enabled }}'
-e CONFIG_kamon_statsd_hostname='{{ metrics.kamon.host }}'
-e CONFIG_kamon_statsd_port='{{ metrics.kamon.port }}'
diff --git a/common/scala/src/main/scala/whisk/common/Logging.scala
b/common/scala/src/main/scala/whisk/common/Logging.scala
index 79ad731..1780211 100644
--- a/common/scala/src/main/scala/whisk/common/Logging.scala
+++ b/common/scala/src/main/scala/whisk/common/Logging.scala
@@ -137,7 +137,7 @@ class PrintStreamLogging(outputStream: PrintStream =
Console.out) extends Loggin
*/
case class LogMarker(token: LogMarkerToken, deltaToTransactionStart: Long,
deltaToMarkerStart: Option[Long] = None) {
override def toString() = {
- val parts = Seq(LogMarker.keyword, token.toString,
deltaToTransactionStart) ++ deltaToMarkerStart
+ val parts = Seq(LogMarker.keyword, token.toStringWithSubAction,
deltaToTransactionStart) ++ deltaToMarkerStart
"[" + parts.mkString(":") + "]"
}
}
@@ -164,27 +164,51 @@ private object Logging {
if (simpleName.endsWith("$")) simpleName.dropRight(1)
else simpleName
}
-
}
private object Emitter {
val timeFormat =
DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'").withZone(ZoneId.of("UTC"))
}
-case class LogMarkerToken(component: String, action: String, state: String) {
- override def toString() = component + "_" + action + "_" + state
+/**
+ * Used to record log message and make a metric name.
+ *
+ * @param component Component like invoker, controller, and docker. It is
defined in LoggingMarkers.
+ * @param action Action of the component.
+ * @param state State of the action.
+ * @param subAction more specific identifier for "action", like `runc.resume`
+ * @param tags tags can be used for whatever granularity you might need.
+ */
+case class LogMarkerToken(component: String,
+ action: String,
+ state: String,
+ subAction: Option[String] = None,
+ tags: Map[String, String] = Map.empty) {
+
+ override def toString = component + "_" + action + "_" + state
+ def toStringWithSubAction =
+ subAction.map(sa => component + "_" + action + "." + sa + "_" +
state).getOrElse(toString)
def asFinish = copy(state = LoggingMarkers.finish)
def asError = copy(state = LoggingMarkers.error)
}
object LogMarkerToken {
- def parse(s: String) = {
+
+ def parse(string: String) = {
// Per convention the components are guaranteed to not contain '_'
// thus it's safe to split at '_' to get the components
- val Array(component, action, state) = s.split("_")
- LogMarkerToken(component, action, state)
+ val Array(component, action, state) = string.split('_')
+
+ val (generalAction, subAction) = action.split('.').toList match {
+ case Nil => throw new IllegalArgumentException("LogMarkerToken
malformed")
+ case a :: Nil => (a, None)
+ case a :: s :: _ => (a, Some(s))
+ }
+
+ LogMarkerToken(component, generalAction, state, subAction)
}
+
}
object MetricEmitter {
@@ -193,20 +217,27 @@ object MetricEmitter {
def emitCounterMetric(token: LogMarkerToken): Unit = {
if (TransactionId.metricsKamon) {
- metrics
- .counter(token.toString)
- .increment(1)
+ if (TransactionId.metricsKamonTags) {
+ metrics
+ .counter(token.toString, token.tags)
+ .increment(1)
+ } else {
+ metrics.counter(token.toStringWithSubAction).increment(1)
+ }
}
}
def emitHistogramMetric(token: LogMarkerToken, value: Long): Unit = {
if (TransactionId.metricsKamon) {
- metrics
- .histogram(token.toString)
- .record(value)
+ if (TransactionId.metricsKamonTags) {
+ metrics
+ .histogram(token.toString, token.tags)
+ .record(value)
+ } else {
+ metrics.histogram(token.toStringWithSubAction).record(value)
+ }
}
}
-
}
object LoggingMarkers {
@@ -259,11 +290,11 @@ object LoggingMarkers {
// Time in invoker
val INVOKER_ACTIVATION = LogMarkerToken(invoker, activation, start)
- def INVOKER_DOCKER_CMD(cmd: String) = LogMarkerToken(invoker,
s"docker.$cmd", start)
- def INVOKER_RUNC_CMD(cmd: String) = LogMarkerToken(invoker, s"runc.$cmd",
start)
+ def INVOKER_DOCKER_CMD(cmd: String) = LogMarkerToken(invoker, "docker",
start, Some(cmd), Map("cmd" -> cmd))
+ def INVOKER_RUNC_CMD(cmd: String) = LogMarkerToken(invoker, "runc", start,
Some(cmd), Map("cmd" -> cmd))
+ def INVOKER_KUBECTL_CMD(cmd: String) = LogMarkerToken(invoker, "kubectl",
start, Some(cmd), Map("cmd" -> cmd))
def INVOKER_CONTAINER_START(containerState: String) =
- LogMarkerToken(invoker, s"container_start_${containerState}", count)
- def INVOKER_KUBECTL_CMD(cmd: String) = LogMarkerToken(invoker,
s"kubectl.$cmd", start)
+ LogMarkerToken(invoker, "containerStart", count, Some(containerState),
Map("containerState" -> containerState))
// Kafka related markers
def KAFKA_QUEUE(topic: String) = LogMarkerToken(kafka, topic, count)
diff --git a/common/scala/src/main/scala/whisk/common/TransactionId.scala
b/common/scala/src/main/scala/whisk/common/TransactionId.scala
index 9b165d0..1f901a1 100644
--- a/common/scala/src/main/scala/whisk/common/TransactionId.scala
+++ b/common/scala/src/main/scala/whisk/common/TransactionId.scala
@@ -107,8 +107,7 @@ case class TransactionId private (meta:
TransactionMetadata) extends AnyVal {
logLevel: LogLevel = DebugLevel,
endTime: Instant = Instant.now(Clock.systemUTC))(implicit
logging: Logging) = {
- val endMarker =
- LogMarkerToken(startMarker.startMarker.component,
startMarker.startMarker.action, LoggingMarkers.finish)
+ val endMarker = startMarker.startMarker.asFinish
val deltaToEnd = deltaToMarker(startMarker, endTime)
if (TransactionId.metricsLog) {
@@ -137,8 +136,7 @@ case class TransactionId private (meta:
TransactionMetadata) extends AnyVal {
def failed(from: AnyRef, startMarker: StartMarker, message: => String = "",
logLevel: LogLevel = WarningLevel)(
implicit logging: Logging) = {
- val endMarker =
- LogMarkerToken(startMarker.startMarker.component,
startMarker.startMarker.action, LoggingMarkers.error)
+ val endMarker = startMarker.startMarker.asError
val deltaToEnd = deltaToMarker(startMarker)
if (TransactionId.metricsLog) {
@@ -199,6 +197,7 @@ object TransactionId {
// get the metric parameters directly from the environment since WhiskConfig
can not be instantiated here
val metricsKamon: Boolean =
sys.env.get("METRICS_KAMON").getOrElse("False").toBoolean
+ val metricsKamonTags: Boolean =
sys.env.get("METRICS_KAMON_TAGS").getOrElse("False").toBoolean
val metricsLog: Boolean =
sys.env.get("METRICS_LOG").getOrElse("True").toBoolean
val unknown = TransactionId(0)
diff --git a/common/scala/src/main/scala/whisk/http/BasicHttpService.scala
b/common/scala/src/main/scala/whisk/http/BasicHttpService.scala
index 46b95e0..e0c570c 100644
--- a/common/scala/src/main/scala/whisk/http/BasicHttpService.scala
+++ b/common/scala/src/main/scala/whisk/http/BasicHttpService.scala
@@ -121,7 +121,13 @@ trait BasicHttpService extends Directives with
TransactionCounter {
val name = "BasicHttpService"
- val token = LogMarkerToken("http",
s"${m.toLowerCase}.${res.status.intValue}", LoggingMarkers.count)
+ val token =
+ LogMarkerToken(
+ "http",
+ m.toLowerCase,
+ LoggingMarkers.count,
+ Some(res.status.intValue.toString),
+ Map("statusCode" -> res.status.intValue.toString))
val marker = LogMarker(token, tid.deltaToStart, Some(tid.deltaToStart))
MetricEmitter.emitHistogramMetric(token, tid.deltaToStart)
diff --git a/docs/metrics.md b/docs/metrics.md
index 5d4fcc0..00544e2 100644
--- a/docs/metrics.md
+++ b/docs/metrics.md
@@ -18,6 +18,12 @@ There are four configurations options available:
Enable/disable whther metric information is send the configured statsd
server.
+- **metrics_kamon_tags: false** [true / false (default: false)]
+
+ Enable/disable whether to use the Kamon tags when sending metrics.
+
+ *Notice: Tag is supported in some kamon-backend. (OpenTSDB, Datadog,
InfluxDB)*
+
- **metrics_kamon_statsd_host** [hostname or ip address]
Hostname or ip address of the statsd server
@@ -31,6 +37,7 @@ Example configuration:
```
metrics_kamon: true
+metrics_kamon_tags: false
metrics_kamon_statsd_host: '192.168.99.100'
metrics_kamon_statsd_port: '8125'
metrics_log: true
diff --git
a/tests/src/test/scala/whisk/core/containerpool/docker/test/DockerClientTests.scala
b/tests/src/test/scala/whisk/core/containerpool/docker/test/DockerClientTests.scala
index 267865d..9093879 100644
---
a/tests/src/test/scala/whisk/core/containerpool/docker/test/DockerClientTests.scala
+++
b/tests/src/test/scala/whisk/core/containerpool/docker/test/DockerClientTests.scala
@@ -288,10 +288,10 @@ class DockerClientTests
logLines.head should include((Seq(dockerCommand, cmd) ++
args).mkString(" "))
val start = LogMarker.parse(logLines.head)
- start.token shouldBe INVOKER_DOCKER_CMD(cmd)
+ start.token.toStringWithSubAction shouldBe
INVOKER_DOCKER_CMD(cmd).toStringWithSubAction
val end = LogMarker.parse(logLines.last)
- end.token shouldBe INVOKER_DOCKER_CMD(cmd).asFinish
+ end.token.toStringWithSubAction shouldBe
INVOKER_DOCKER_CMD(cmd).asFinish.toStringWithSubAction
stream.reset()
result
@@ -320,10 +320,10 @@ class DockerClientTests
a[RuntimeException] should be thrownBy await(f)
val start = LogMarker.parse(logLines.head)
- start.token shouldBe INVOKER_DOCKER_CMD(cmd)
+ start.token.toStringWithSubAction shouldBe
INVOKER_DOCKER_CMD(cmd).toStringWithSubAction
val end = LogMarker.parse(logLines.last)
- end.token shouldBe INVOKER_DOCKER_CMD(cmd).asError
+ end.token.toStringWithSubAction shouldBe
INVOKER_DOCKER_CMD(cmd).asError.toStringWithSubAction
stream.reset()
}
diff --git
a/tests/src/test/scala/whisk/core/containerpool/docker/test/RuncClientTests.scala
b/tests/src/test/scala/whisk/core/containerpool/docker/test/RuncClientTests.scala
index cf5990b..340b15d 100644
---
a/tests/src/test/scala/whisk/core/containerpool/docker/test/RuncClientTests.scala
+++
b/tests/src/test/scala/whisk/core/containerpool/docker/test/RuncClientTests.scala
@@ -70,12 +70,12 @@ class RuncClientTests extends FlatSpec with Matchers with
StreamLogging with Bef
// start log maker must be found
val start = LogMarker.parse(logLines.head)
- start.token should be(INVOKER_RUNC_CMD(cmd))
+ start.token.toStringWithSubAction should
be(INVOKER_RUNC_CMD(cmd).toStringWithSubAction)
// end log marker must be found
val expectedEnd = if (failed) INVOKER_RUNC_CMD(cmd).asError else
INVOKER_RUNC_CMD(cmd).asFinish
val end = LogMarker.parse(logLines.last)
- end.token shouldBe expectedEnd
+ end.token.toStringWithSubAction shouldBe expectedEnd.toStringWithSubAction
}
behavior of "RuncClient"
--
To stop receiving notification emails like this one, please contact
[email protected].
