Repository: incubator-griffin Updated Branches: refs/heads/master 5c34a83f1 -> b4f7d1551
[GRIFFIN-212] Add matchedFraction in streaming accuracy metrics Author: Lionel Liu <bhlx3l...@163.com> Closes #455 from bhlx3lyx7/util-update-0.3. Project: http://git-wip-us.apache.org/repos/asf/incubator-griffin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-griffin/commit/b4f7d155 Tree: http://git-wip-us.apache.org/repos/asf/incubator-griffin/tree/b4f7d155 Diff: http://git-wip-us.apache.org/repos/asf/incubator-griffin/diff/b4f7d155 Branch: refs/heads/master Commit: b4f7d1551cc4165a81f9cf9340b50c3099ee0f1b Parents: 5c34a83 Author: Lionel Liu <bhlx3l...@163.com> Authored: Mon Nov 19 16:57:57 2018 +0800 Committer: William Guo <gu...@apache.org> Committed: Mon Nov 19 16:57:57 2018 +0800 ---------------------------------------------------------------------- .../measure/context/streaming/metric/AccuracyMetric.scala | 4 +++- .../apache/griffin/measure/step/transform/DataFrameOps.scala | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b4f7d155/measure/src/main/scala/org/apache/griffin/measure/context/streaming/metric/AccuracyMetric.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/context/streaming/metric/AccuracyMetric.scala b/measure/src/main/scala/org/apache/griffin/measure/context/streaming/metric/AccuracyMetric.scala index 19dfb9e..5065417 100644 --- a/measure/src/main/scala/org/apache/griffin/measure/context/streaming/metric/AccuracyMetric.scala +++ b/measure/src/main/scala/org/apache/griffin/measure/context/streaming/metric/AccuracyMetric.scala @@ -51,6 +51,8 @@ case class AccuracyMetric(miss: Long, total: Long) extends Metric { def getMatch: Long = total - miss - def matchPercentage: Double = if (getTotal <= 0) 0 else getMatch.toDouble / getTotal * 100 + def matchFraction: Double = if (getTotal <= 0) 1 else getMatch.toDouble / getTotal + + def matchPercentage: Double = matchFraction * 100 } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b4f7d155/measure/src/main/scala/org/apache/griffin/measure/step/transform/DataFrameOps.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/step/transform/DataFrameOps.scala b/measure/src/main/scala/org/apache/griffin/measure/step/transform/DataFrameOps.scala index 088f328..ba64d33 100644 --- a/measure/src/main/scala/org/apache/griffin/measure/step/transform/DataFrameOps.scala +++ b/measure/src/main/scala/org/apache/griffin/measure/step/transform/DataFrameOps.scala @@ -21,8 +21,7 @@ package org.apache.griffin.measure.step.transform import java.util.Date import org.apache.spark.sql.{Encoders, Row, SQLContext, _} -import org.apache.spark.sql.types.{BooleanType, LongType, StructField, StructType} - +import org.apache.spark.sql.types._ import org.apache.griffin.measure.context.ContextId import org.apache.griffin.measure.context.streaming.metric._ import org.apache.griffin.measure.context.streaming.metric.CacheResults.CacheResult @@ -43,6 +42,7 @@ object DataFrameOps { val _miss = "miss" val _total = "total" val _matched = "matched" + val _matchedFraction = "matchedFraction" } def fromJson(sqlContext: SQLContext, @@ -70,6 +70,7 @@ object DataFrameOps { val miss = details.getStringOrKey(_miss) val total = details.getStringOrKey(_total) val matched = details.getStringOrKey(_matched) + val matchedFraction = details.getStringOrKey(_matchedFraction) val updateTime = new Date().getTime @@ -107,12 +108,13 @@ object DataFrameOps { StructField(miss, LongType), StructField(total, LongType), StructField(matched, LongType), + StructField(matchedFraction, DoubleType), StructField(ConstantColumns.record, BooleanType), StructField(ConstantColumns.empty, BooleanType) )) val rows = updatedResults.map { r => val ar = r.result.asInstanceOf[AccuracyMetric] - Row(r.timeStamp, ar.miss, ar.total, ar.getMatch, !ar.initial, ar.eventual) + Row(r.timeStamp, ar.miss, ar.total, ar.getMatch, ar.matchFraction, !ar.initial, ar.eventual) }.toArray val rowRdd = sqlContext.sparkContext.parallelize(rows) val retDf = sqlContext.createDataFrame(rowRdd, schema)