This is an automated email from the ASF dual-hosted git repository.
xhsun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 367f7e4 [TE] Merge anomaly properties (#4744)
367f7e4 is described below
commit 367f7e4743f00daebb4c67dd9e9c0f31e62b386f
Author: Xiaohui Sun <[email protected]>
AuthorDate: Tue Oct 29 21:04:50 2019 -0700
[TE] Merge anomaly properties (#4744)
---
.../app/pods/components/alert-details/component.js | 18 ++++-
.../thirdeye/detection/algorithm/MergeWrapper.java | 6 +-
.../wrapper/ChildKeepingMergeWrapper.java | 4 ++
.../apache/pinot/thirdeye/util/ThirdEyeUtils.java | 82 ++++++++++++++++------
.../pinot/thirdeye/util/ThirdEyeUtilsTest.java | 20 ++++++
5 files changed, 104 insertions(+), 26 deletions(-)
diff --git
a/thirdeye/thirdeye-frontend/app/pods/components/alert-details/component.js
b/thirdeye/thirdeye-frontend/app/pods/components/alert-details/component.js
index 222a81b..7b3294c 100644
--- a/thirdeye/thirdeye-frontend/app/pods/components/alert-details/component.js
+++ b/thirdeye/thirdeye-frontend/app/pods/components/alert-details/component.js
@@ -1046,9 +1046,25 @@ export default Component.extend({
return isMetricNew;
},
+ _formattedRule(properties) {
+ let result;
+ if (properties && typeof properties === 'object') {
+ if (properties.detectorComponentName) {
+ // The format is rule1_name:rule1_type,rule2_name:rule2_type ...
+ // For example:
wow_10_percent_change:PERCENTAGE_RULE,algorithm:ALGORITHM
+ let rules = [];
+ properties.detectorComponentName.split(',').forEach(x => {
rules.push(x.split(':')[0])});
+ result = rules.sort().join();
+ } else {
+ result = '--';
+ }
+ }
+ return result;
+ },
+
_fetchAnomalies() {
set(this, 'getAnomaliesError', false);
-
+
// If the user is running the detection with a new metric, we should reset
the state of time series and anomalies for comparison
if (this._checkMetricIfCreateAlertPreview()) {
this.setProperties({
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/algorithm/MergeWrapper.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/algorithm/MergeWrapper.java
index f3b6401..f41fe2b 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/algorithm/MergeWrapper.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/algorithm/MergeWrapper.java
@@ -31,7 +31,6 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
import org.apache.commons.collections4.MapUtils;
import org.apache.pinot.thirdeye.anomaly.AnomalyType;
import org.apache.pinot.thirdeye.common.dimension.DimensionMap;
@@ -45,6 +44,7 @@ import
org.apache.pinot.thirdeye.detection.DetectionPipelineResult;
import org.apache.pinot.thirdeye.detection.DetectionUtils;
import org.apache.pinot.thirdeye.detection.PredictionResult;
import org.apache.pinot.thirdeye.detection.spi.model.AnomalySlice;
+import org.apache.pinot.thirdeye.util.ThirdEyeUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -204,9 +204,7 @@ public class MergeWrapper extends DetectionPipeline {
parent.setEndTime(Math.max(parent.getEndTime(), anomaly.getEndTime()));
// merge the anomaly's properties into parent
- Map<String, String> properties = parent.getProperties();
- properties.putAll(anomaly.getProperties());
- parent.setProperties(properties);
+ ThirdEyeUtils.mergeAnomalyProperties(parent.getProperties(),
anomaly.getProperties());
if (isExistingAnomaly(parent)) {
modifiedExistingAnomalies.add(parent);
} else {
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
index 1711aeb..e78cba0 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
@@ -31,6 +31,7 @@ import
org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
import org.apache.pinot.thirdeye.detection.DataProvider;
import org.apache.pinot.thirdeye.detection.algorithm.MergeWrapper;
+import org.apache.pinot.thirdeye.util.ThirdEyeUtils;
/**
@@ -95,6 +96,9 @@ public class ChildKeepingMergeWrapper extends
BaselineFillingMergeWrapper {
}
parent.setEndTime(Math.max(parent.getEndTime(), anomaly.getEndTime()));
+ // merge the anomaly's properties into parent
+ ThirdEyeUtils.mergeAnomalyProperties(parent.getProperties(),
anomaly.getProperties());
+
if (anomaly.getChildren().isEmpty()) {
parent.getChildren().add(anomaly);
} else {
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/util/ThirdEyeUtils.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/util/ThirdEyeUtils.java
index 52ff174..ad6bdf3 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/util/ThirdEyeUtils.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/util/ThirdEyeUtils.java
@@ -19,6 +19,9 @@
package org.apache.pinot.thirdeye.util;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
@@ -26,15 +29,10 @@ import com.google.common.cache.LoadingCache;
import com.google.common.cache.RemovalListener;
import com.google.common.cache.RemovalNotification;
import com.google.common.cache.Weigher;
+import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
import io.dropwizard.configuration.YamlConfigurationFactory;
-import java.util.concurrent.TimeUnit;
-import org.apache.commons.io.FileUtils;
-import org.apache.pinot.common.data.TimeGranularitySpec.TimeFormat;
-import org.apache.pinot.thirdeye.common.dimension.DimensionMap;
-
-import org.apache.pinot.thirdeye.dashboard.ThirdEyeDashboardConfiguration;
-import org.apache.pinot.thirdeye.datalayer.util.DaoProviderUtil;
import io.dropwizard.jackson.Jackson;
import java.io.File;
import java.io.IOException;
@@ -48,37 +46,38 @@ import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ExecutionException;
-
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import javax.validation.Validation;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
-import org.apache.pinot.thirdeye.datasource.RelationalQuery;
-import org.apache.pinot.thirdeye.datasource.pinot.resultset.ThirdEyeResultSet;
-import
org.apache.pinot.thirdeye.datasource.pinot.resultset.ThirdEyeResultSetGroup;
-import org.joda.time.Period;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.Multimap;
+import org.apache.pinot.common.data.TimeGranularitySpec.TimeFormat;
+import org.apache.pinot.thirdeye.common.dimension.DimensionMap;
import org.apache.pinot.thirdeye.common.time.TimeGranularity;
import org.apache.pinot.thirdeye.common.time.TimeSpec;
+import org.apache.pinot.thirdeye.dashboard.ThirdEyeDashboardConfiguration;
import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager;
import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO;
import org.apache.pinot.thirdeye.datalayer.pojo.AlertConfigBean.COMPARE_MODE;
-import org.apache.pinot.thirdeye.datalayer.pojo.DatasetConfigBean;
import org.apache.pinot.thirdeye.datalayer.pojo.MetricConfigBean;
+import org.apache.pinot.thirdeye.datalayer.util.DaoProviderUtil;
import org.apache.pinot.thirdeye.datasource.DAORegistry;
import org.apache.pinot.thirdeye.datasource.MetricExpression;
import org.apache.pinot.thirdeye.datasource.MetricFunction;
+import org.apache.pinot.thirdeye.datasource.RelationalQuery;
import org.apache.pinot.thirdeye.datasource.ThirdEyeCacheRegistry;
import org.apache.pinot.thirdeye.datasource.cache.MetricDataset;
+import org.apache.pinot.thirdeye.datasource.pinot.resultset.ThirdEyeResultSet;
+import
org.apache.pinot.thirdeye.datasource.pinot.resultset.ThirdEyeResultSetGroup;
+import org.joda.time.Period;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.pinot.thirdeye.detection.wrapper.GrouperWrapper.*;
public abstract class ThirdEyeUtils {
@@ -619,4 +618,45 @@ public abstract class ThirdEyeUtils {
}
return (jvmMaxMemoryInBytes / 102400) * percentage;
}
+
+
+ /**
+ * Merge child's properties into parent's properties.
+ * If the property exists in both then use parent's property.
+ * For property = "detectorComponentName", combine the parent and child.
+ * @param parent The parent anomaly's properties.
+ * @param child The child anomaly's properties.
+ */
+ public static void mergeAnomalyProperties(Map<String, String> parent,
Map<String, String> child) {
+ for (String key : child.keySet()) {
+ if (!parent.containsKey(key)) {
+ parent.put(key, child.get(key));
+ } else {
+ // combine detectorComponentName
+ if (key.equals(PROP_DETECTOR_COMPONENT_NAME)) {
+ String component =
ThirdEyeUtils.combineComponents(parent.get(PROP_DETECTOR_COMPONENT_NAME),
child.get(PROP_DETECTOR_COMPONENT_NAME));
+ parent.put(PROP_DETECTOR_COMPONENT_NAME, component);
+ }
+ }
+ }
+ }
+
+ /**
+ * Combine two components with comma separated.
+ * For example, will combine "component1" and "component2" into "component1,
component2".
+ *
+ * @param component1 The first component.
+ * @param component2 The second component.
+ * @return The combined components.
+ */
+ private static String combineComponents(String component1, String
component2) {
+ List<String> components = new ArrayList<>();
+ for (String component : component1.split(",")) {
+ components.add(component);
+ }
+ for (String component : component2.split(",")) {
+ components.add(component);
+ }
+ return String.join(",",
components.stream().distinct().collect(Collectors.toList()));
+ }
}
diff --git
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/util/ThirdEyeUtilsTest.java
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/util/ThirdEyeUtilsTest.java
index 87c9d91..5fc8ab3 100644
---
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/util/ThirdEyeUtilsTest.java
+++
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/util/ThirdEyeUtilsTest.java
@@ -17,7 +17,9 @@
package org.apache.pinot.thirdeye.util;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -173,4 +175,22 @@ public class ThirdEyeUtilsTest {
Assert.assertFalse(errorMessage.contains("e3"));
}
+ @Test
+ public void testMergeAnomalyProperties() {
+ Map<String, String> parentProperties = new HashMap<>();
+ parentProperties.put("p1", "value1");
+ parentProperties.put("p2", "value2");
+ parentProperties.put("detectorComponentName", "rule1");
+
+ Map<String, String> childProperties = new HashMap<>();
+ childProperties.put("p1", "value3");
+ childProperties.put("c1", "value4");
+ childProperties.put("detectorComponentName", "rule2");
+
+ ThirdEyeUtils.mergeAnomalyProperties(parentProperties, childProperties);
+ Assert.assertEquals(parentProperties.get("p1"), "value1");
+ Assert.assertEquals(parentProperties.get("p2"), "value2");
+ Assert.assertEquals(parentProperties.get("c1"), "value4");
+ Assert.assertEquals(parentProperties.get("detectorComponentName"),
"rule1,rule2");
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
