Repository: metron Updated Branches: refs/heads/feature/METRON-1416-upgrade-solr 0717cfc25 -> 476856192
Merge branch 'master' into feature/METRON-1416-upgrade-solr Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e7233b0f Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e7233b0f Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e7233b0f Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: e7233b0f82f372e6b1339de56a115fa7e4581f07 Parents: 4170887 Author: cstella <ceste...@gmail.com> Authored: Mon Jun 11 21:41:06 2018 -0400 Committer: cstella <ceste...@gmail.com> Committed: Tue Jun 12 09:46:46 2018 -0400 ---------------------------------------------------------------------- .../org/apache/metron/common/Constants.java | 1 + .../elasticsearch/dao/ElasticsearchDao.java | 4 ++ .../dao/ElasticsearchMetaAlertDao.java | 30 +++++++++----- .../dao/ElasticsearchMetaAlertUpdateDao.java | 4 +- .../indexing/dao/metaalert/MetaAlertConfig.java | 43 ++++++++++++-------- .../AbstractLuceneMetaAlertUpdateDao.java | 3 ++ .../dao/metaalert/MetaAlertIntegrationTest.java | 2 +- .../AbstractLuceneMetaAlertUpdateDaoTest.java | 28 +++++++++---- .../metron/solr/dao/SolrMetaAlertDao.java | 29 +++++++++---- .../metron/solr/dao/SolrMetaAlertSearchDao.java | 15 ++++--- .../metron/solr/dao/SolrMetaAlertUpdateDao.java | 2 +- .../SolrMetaAlertIntegrationTest.java | 28 +++++++++---- 12 files changed, 129 insertions(+), 60 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/e7233b0f/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java index f74660c..4a8bea2 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java @@ -28,6 +28,7 @@ public class Constants { public static final long DEFAULT_CONFIGURED_BOLT_TIMEOUT = 5000; public static final String SENSOR_TYPE = "source.type"; public static final String SENSOR_TYPE_FIELD_PROPERTY = "source.type.field"; + public static final String THREAT_SCORE_FIELD_PROPERTY = "threat.triage.score.field"; public static final String ENRICHMENT_TOPIC = "enrichments"; public static final String INDEXING_TOPIC = "indexing"; public static final String ERROR_STREAM = "error"; http://git-wip-us.apache.org/repos/asf/metron/blob/e7233b0f/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java index eae0a39..3eb86ce 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java @@ -85,6 +85,10 @@ public class ElasticsearchDao implements IndexDao { //uninitialized. } + public AccessConfig getAccessConfig() { + return accessConfig; + } + @Override public synchronized void init(AccessConfig config) { if (this.client == null) { http://git-wip-us.apache.org/repos/asf/metron/blob/e7233b0f/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java index f73a640..649077e 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java @@ -68,6 +68,7 @@ import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.util.function.Supplier; import java.util.stream.Collectors; import static org.apache.metron.common.Constants.GUID; @@ -83,10 +84,8 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao { public static final String THREAT_TRIAGE_FIELD = MetaAlertConstants.THREAT_FIELD_DEFAULT .replace('.', ':'); public static final String METAALERTS_INDEX = "metaalert_index"; - public static final String SOURCE_TYPE_FIELD = Constants.SENSOR_TYPE.replace('.', ':'); protected String metaAlertsIndex = METAALERTS_INDEX; - protected String threatTriageField = THREAT_TRIAGE_FIELD; protected String threatSort = MetaAlertConstants.THREAT_SORT_DEFAULT; private ElasticsearchDao elasticsearchDao; @@ -102,8 +101,7 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao { * @param indexDao The Dao to wrap */ public ElasticsearchMetaAlertDao(IndexDao indexDao) { - this(indexDao, METAALERTS_INDEX, MetaAlertConstants.THREAT_FIELD_DEFAULT, - MetaAlertConstants.THREAT_SORT_DEFAULT); + this(indexDao, METAALERTS_INDEX, MetaAlertConstants.THREAT_SORT_DEFAULT); } /** @@ -114,10 +112,8 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao { * can be either max, min, average, count, median, or sum. */ public ElasticsearchMetaAlertDao(IndexDao indexDao, String metaAlertsIndex, - String triageLevelField, String threatSort) { init(indexDao, Optional.of(threatSort)); - this.threatTriageField = triageLevelField; this.threatSort = threatSort; this.metaAlertsIndex = metaAlertsIndex; } @@ -158,13 +154,25 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao { if (threatSort.isPresent()) { this.threatSort = threatSort.get(); } - + Supplier<Map<String, Object>> globalConfigSupplier = () -> new HashMap<>(); + if(elasticsearchDao != null && elasticsearchDao.getAccessConfig() != null) { + globalConfigSupplier = elasticsearchDao.getAccessConfig().getGlobalConfigSupplier(); + } MetaAlertConfig config = new MetaAlertConfig( metaAlertsIndex, - threatTriageField, - this.threatSort, - ElasticsearchMetaAlertDao.SOURCE_TYPE_FIELD - ); + this.threatSort, + globalConfigSupplier + ) { + @Override + protected String getDefaultThreatTriageField() { + return THREAT_TRIAGE_FIELD; + } + + @Override + protected String getDefaultSourceTypeField() { + return SOURCE_TYPE_FIELD; + } + }; this.metaAlertSearchDao = new ElasticsearchMetaAlertSearchDao( elasticsearchDao, http://git-wip-us.apache.org/repos/asf/metron/blob/e7233b0f/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java index d3bdcbb..d757dfe 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java @@ -29,6 +29,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Supplier; import java.util.stream.Collectors; import org.apache.lucene.search.join.ScoreMode; import org.apache.metron.common.Constants; @@ -37,6 +38,7 @@ import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig; import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants; import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest; import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse; +import org.apache.metron.indexing.dao.metaalert.MetaAlertDao; import org.apache.metron.indexing.dao.metaalert.MetaAlertRetrieveLatestDao; import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus; import org.apache.metron.indexing.dao.metaalert.MetaScores; @@ -96,7 +98,7 @@ public class ElasticsearchMetaAlertUpdateDao extends AbstractLuceneMetaAlertUpda getConfig().getThreatSort()); // Add source type to be consistent with other sources and allow filtering metaAlert.getDocument() - .put(ElasticsearchMetaAlertDao.SOURCE_TYPE_FIELD, MetaAlertConstants.METAALERT_TYPE); + .put(getConfig().getSourceTypeField(), MetaAlertConstants.METAALERT_TYPE); // Start a list of updates / inserts we need to run Map<Document, Optional<String>> updates = new HashMap<>(); http://git-wip-us.apache.org/repos/asf/metron/blob/e7233b0f/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConfig.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConfig.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConfig.java index 9254425..b538bc2 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConfig.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConfig.java @@ -18,26 +18,30 @@ package org.apache.metron.indexing.dao.metaalert; -public class MetaAlertConfig { +import org.apache.metron.common.Constants; +import org.apache.metron.common.configuration.ConfigurationsUtils; + +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; + +public abstract class MetaAlertConfig { private String metaAlertIndex; - private String threatTriageField; private String threatSort; - private String sourceTypeField; + private Supplier<Map<String, Object>> globalConfigSupplier; /** * Simple object for storing and retrieving configs, primarily to make passing all the info to * the sub DAOs easier. * @param metaAlertIndex The metaalert index or collection we're using - * @param threatTriageField The threat triage field's name * @param threatSort The sorting operation on the threat triage field - * @param sourceTypeField The source type field */ - public MetaAlertConfig(String metaAlertIndex, String threatTriageField, - String threatSort, String sourceTypeField) { + public MetaAlertConfig( String metaAlertIndex + , String threatSort + , Supplier<Map<String, Object>> globalConfigSupplier) { this.metaAlertIndex = metaAlertIndex; - this.threatTriageField = threatTriageField; this.threatSort = threatSort; - this.sourceTypeField = sourceTypeField; + this.globalConfigSupplier = globalConfigSupplier; } public String getMetaAlertIndex() { @@ -49,12 +53,14 @@ public class MetaAlertConfig { } public String getThreatTriageField() { - return threatTriageField; + Optional<Map<String, Object>> globalConfig = Optional.ofNullable(globalConfigSupplier.get()); + if(!globalConfig.isPresent()) { + return getDefaultThreatTriageField(); + } + return ConfigurationsUtils.getFieldName(globalConfig.get(), Constants.THREAT_SCORE_FIELD_PROPERTY, getDefaultThreatTriageField()); } - public void setThreatTriageField(String threatTriageField) { - this.threatTriageField = threatTriageField; - } + protected abstract String getDefaultThreatTriageField(); public String getThreatSort() { return threatSort; @@ -65,10 +71,13 @@ public class MetaAlertConfig { } public String getSourceTypeField() { - return sourceTypeField; + Optional<Map<String, Object>> globalConfig = Optional.ofNullable(globalConfigSupplier.get()); + if(!globalConfig.isPresent()) { + return getDefaultSourceTypeField(); + } + return ConfigurationsUtils.getFieldName(globalConfig.get(), Constants.SENSOR_TYPE_FIELD_PROPERTY, getDefaultSourceTypeField()); } - public void setSourceTypeField(String sourceTypeField) { - this.sourceTypeField = sourceTypeField; - } + protected abstract String getDefaultSourceTypeField(); + } http://git-wip-us.apache.org/repos/asf/metron/blob/e7233b0f/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java index b47d648..4d48075 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java @@ -30,8 +30,10 @@ import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.util.function.Supplier; import java.util.stream.Collectors; import org.apache.metron.common.Constants; +import org.apache.metron.common.configuration.ConfigurationsUtils; import org.apache.metron.indexing.dao.RetrieveLatestDao; import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig; import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants; @@ -331,4 +333,5 @@ public abstract class AbstractLuceneMetaAlertUpdateDao implements MetaAlertUpdat updateDao.batchUpdate(updates); } // else we have no updates, so don't do anything } + } http://git-wip-us.apache.org/repos/asf/metron/blob/e7233b0f/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java index b4f7d38..6f96fb5 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java @@ -111,7 +111,7 @@ public abstract class MetaAlertIntegrationTest { }, { "op": "add", - "path": "/alert", + "path": "/metron_alert", "value": [] } ], http://git-wip-us.apache.org/repos/asf/metron/blob/e7233b0f/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java index 7028b75..5a70636 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java @@ -43,6 +43,8 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Optional; import java.util.UUID; + +import com.google.common.collect.ImmutableMap; import org.adrianwalker.multilinestring.Multiline; import org.apache.commons.math.util.MathUtils; import org.apache.metron.common.Constants; @@ -88,12 +90,24 @@ public class AbstractLuceneMetaAlertUpdateDaoTest { private static final String METAALERT_INDEX = "metaalert_index"; private static final String METAALERT_GUID = "meta_0"; private static final String DEFAULT_PREFIX = "child_"; - private static final MetaAlertConfig TEST_CONFIG = new MetaAlertConfig( - METAALERT_INDEX, - THREAT_FIELD_DEFAULT, - THREAT_SORT_DEFAULT, - Constants.SENSOR_TYPE - ); + private static final MetaAlertConfig TEST_CONFIG = + new MetaAlertConfig(METAALERT_INDEX + , THREAT_SORT_DEFAULT + , () -> ImmutableMap.of(Constants.SENSOR_TYPE_FIELD_PROPERTY, Constants.SENSOR_TYPE + , Constants.THREAT_SCORE_FIELD_PROPERTY, THREAT_FIELD_DEFAULT + ) + ) { + + @Override + protected String getDefaultThreatTriageField() { + return THREAT_FIELD_DEFAULT.replace(':', '.'); + } + + @Override + protected String getDefaultSourceTypeField() { + return Constants.SENSOR_TYPE; + } + }; private static Map<String, Document> documents = new HashMap<>(); @@ -190,7 +204,7 @@ public class AbstractLuceneMetaAlertUpdateDaoTest { "patch": [ { "op": "add", - "path": "/alert", + "path": "/metron_alert", "value": [] } ], http://git-wip-us.apache.org/repos/asf/metron/blob/e7233b0f/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java index 8b37a49..4748315 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java @@ -19,9 +19,12 @@ package org.apache.metron.solr.dao; import java.io.IOException; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Supplier; + import org.apache.metron.common.Constants; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.IndexDao; @@ -57,7 +60,6 @@ public class SolrMetaAlertDao implements MetaAlertDao { private SolrMetaAlertUpdateDao metaAlertUpdateDao; private SolrMetaAlertRetrieveLatestDao metaAlertRetrieveLatestDao; protected String metaAlertsCollection = METAALERTS_COLLECTION; - protected String threatTriageField = MetaAlertConstants.THREAT_FIELD_DEFAULT; protected String threatSort = MetaAlertConstants.THREAT_SORT_DEFAULT; /** @@ -69,14 +71,12 @@ public class SolrMetaAlertDao implements MetaAlertDao { SolrMetaAlertRetrieveLatestDao metaAlertRetrieveLatestDao) { this(indexDao, metaAlertSearchDao, metaAlertUpdateDao, metaAlertRetrieveLatestDao, METAALERTS_COLLECTION, - MetaAlertConstants.THREAT_FIELD_DEFAULT, MetaAlertConstants.THREAT_SORT_DEFAULT); } /** * Wraps an {@link org.apache.metron.indexing.dao.IndexDao} to handle meta alerts. * @param indexDao The Dao to wrap - * @param triageLevelField The field name to use as the threat scoring field * @param threatSort The summary aggregation of all child threat triage scores used * as the overall threat triage score for the metaalert. This * can be either max, min, average, count, median, or sum. @@ -85,14 +85,12 @@ public class SolrMetaAlertDao implements MetaAlertDao { SolrMetaAlertUpdateDao metaAlertUpdateDao, SolrMetaAlertRetrieveLatestDao metaAlertRetrieveLatestDao, String metaAlertsCollection, - String triageLevelField, String threatSort) { init(indexDao, Optional.of(threatSort)); this.metaAlertSearchDao = metaAlertSearchDao; this.metaAlertUpdateDao = metaAlertUpdateDao; this.metaAlertRetrieveLatestDao = metaAlertRetrieveLatestDao; this.metaAlertsCollection = metaAlertsCollection; - this.threatTriageField = triageLevelField; this.threatSort = threatSort; } @@ -126,16 +124,29 @@ public class SolrMetaAlertDao implements MetaAlertDao { "Need a SolrDao when using SolrMetaAlertDao" ); } + Supplier<Map<String, Object>> globalConfigSupplier = () -> new HashMap<>(); + if(metaAlertSearchDao != null && metaAlertSearchDao.solrSearchDao != null && metaAlertSearchDao.solrSearchDao.getAccessConfig() != null) { + globalConfigSupplier = metaAlertSearchDao.solrSearchDao.getAccessConfig().getGlobalConfigSupplier(); + } MetaAlertConfig config = new MetaAlertConfig( metaAlertsCollection, - threatTriageField, this.threatSort, - Constants.SENSOR_TYPE - ); + globalConfigSupplier + ) { + @Override + protected String getDefaultThreatTriageField() { + return MetaAlertConstants.THREAT_FIELD_DEFAULT.replace(':', '.'); + } + + @Override + protected String getDefaultSourceTypeField() { + return Constants.SENSOR_TYPE; + } + }; SolrClient solrClient = solrDao.getSolrClient(solrDao.getZkHosts()); - this.metaAlertSearchDao = new SolrMetaAlertSearchDao(solrClient, solrDao.getSolrSearchDao()); + this.metaAlertSearchDao = new SolrMetaAlertSearchDao(solrClient, solrDao.getSolrSearchDao(), config); this.metaAlertRetrieveLatestDao = new SolrMetaAlertRetrieveLatestDao(solrDao); this.metaAlertUpdateDao = new SolrMetaAlertUpdateDao( solrDao, http://git-wip-us.apache.org/repos/asf/metron/blob/e7233b0f/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java index c6f7124..c1e3af6 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.apache.metron.common.Constants; +import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig; import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants; import org.apache.metron.indexing.dao.metaalert.MetaAlertSearchDao; import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus; @@ -57,10 +58,12 @@ public class SolrMetaAlertSearchDao implements MetaAlertSearchDao { transient SolrSearchDao solrSearchDao; transient SolrClient solrClient; + private MetaAlertConfig config; - public SolrMetaAlertSearchDao(SolrClient solrClient, SolrSearchDao solrSearchDao) { + public SolrMetaAlertSearchDao(SolrClient solrClient, SolrSearchDao solrSearchDao, MetaAlertConfig config) { this.solrClient = solrClient; this.solrSearchDao = solrSearchDao; + this.config = config; } @Override @@ -76,7 +79,7 @@ public class SolrMetaAlertSearchDao implements MetaAlertSearchDao { MetaAlertConstants.STATUS_FIELD + ":" + MetaAlertStatus.ACTIVE.getStatusString(); String guidClause = Constants.GUID + ":" + guid; String fullClause = "{!parent which=" + activeClause + "}" + guidClause; - String metaalertTypeClause = Constants.SENSOR_TYPE + ":" + MetaAlertConstants.METAALERT_TYPE; + String metaalertTypeClause = config.getSourceTypeField() + ":" + MetaAlertConstants.METAALERT_TYPE; SolrQuery solrQuery = new SolrQuery() .setQuery(fullClause) .setFields("*", "[child parentFilter=" + metaalertTypeClause + " limit=999]") @@ -120,7 +123,7 @@ public class SolrMetaAlertSearchDao implements MetaAlertSearchDao { String activeStatusClause = MetaAlertConstants.STATUS_FIELD + ":" + MetaAlertStatus.ACTIVE.getStatusString(); - String metaalertTypeClause = Constants.SENSOR_TYPE + ":" + MetaAlertConstants.METAALERT_TYPE; + String metaalertTypeClause = config.getSourceTypeField() + ":" + MetaAlertConstants.METAALERT_TYPE; // Use the 'v=' form in order to ensure complex clauses are properly handled. // Per the docs, the 'which=' clause should be used to identify all metaalert parents, not to // filter @@ -157,10 +160,10 @@ public class SolrMetaAlertSearchDao implements MetaAlertSearchDao { // Get them in a second query. // However, we can only retrieve them if we have the source type field (either explicit or // wildcard). - if (fieldList.contains("*") || fieldList.contains(Constants.SENSOR_TYPE)) { + if (fieldList.contains("*") || fieldList.contains(config.getSourceTypeField())) { List<String> metaalertGuids = new ArrayList<>(); for (SearchResult result : results.getResults()) { - if (result.getSource().get(Constants.SENSOR_TYPE) + if (result.getSource().get(config.getSourceTypeField()) .equals(MetaAlertConstants.METAALERT_TYPE)) { // Then we need to add it to the list to retrieve child alerts in a second query. metaalertGuids.add(result.getId()); @@ -201,7 +204,7 @@ public class SolrMetaAlertSearchDao implements MetaAlertSearchDao { @Override public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException { // Make sure to escape any problematic characters here - String sourceType = ClientUtils.escapeQueryChars(Constants.SENSOR_TYPE); + String sourceType = ClientUtils.escapeQueryChars(config.getSourceTypeField()); String baseQuery = groupRequest.getQuery(); String adjustedQuery = baseQuery + " -" + MetaAlertConstants.METAALERT_FIELD + ":[* TO *]" + " -" + sourceType + ":" + MetaAlertConstants.METAALERT_TYPE; http://git-wip-us.apache.org/repos/asf/metron/blob/e7233b0f/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java index b96bbc6..132d872 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java @@ -90,7 +90,7 @@ public class SolrMetaAlertUpdateDao extends AbstractLuceneMetaAlertUpdateDao imp getConfig().getThreatSort()); // Add source type to be consistent with other sources and allow filtering - metaAlert.getDocument().put(Constants.SENSOR_TYPE, MetaAlertConstants.METAALERT_TYPE); + metaAlert.getDocument().put(getConfig().getSourceTypeField(), MetaAlertConstants.METAALERT_TYPE); // Start a list of updates / inserts we need to run Map<Document, Optional<String>> updates = new HashMap<>(); http://git-wip-us.apache.org/repos/asf/metron/blob/e7233b0f/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrMetaAlertIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrMetaAlertIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrMetaAlertIntegrationTest.java index f7dd02c..6687e9a 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrMetaAlertIntegrationTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrMetaAlertIntegrationTest.java @@ -33,6 +33,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; + +import com.google.common.collect.ImmutableMap; import org.apache.metron.common.Constants; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig; @@ -93,16 +95,28 @@ public class SolrMetaAlertIntegrationTest extends MetaAlertIntegrationTest { solrDao = new SolrDao(); solrDao.init(accessConfig); - MetaAlertConfig config = new MetaAlertConfig( - METAALERTS_COLLECTION, - THREAT_FIELD_DEFAULT, - THREAT_SORT_DEFAULT, - Constants.SENSOR_TYPE - ); + MetaAlertConfig config = new MetaAlertConfig(METAALERTS_COLLECTION + , THREAT_SORT_DEFAULT + , () -> ImmutableMap.of(Constants.SENSOR_TYPE_FIELD_PROPERTY, Constants.SENSOR_TYPE + , Constants.THREAT_SCORE_FIELD_PROPERTY, THREAT_FIELD_DEFAULT + ) + ) { + + @Override + protected String getDefaultThreatTriageField() { + return THREAT_FIELD_DEFAULT.replace(':', '.'); + } + + @Override + protected String getDefaultSourceTypeField() { + return Constants.SENSOR_TYPE; + } + }; + SolrMetaAlertSearchDao searchDao = new SolrMetaAlertSearchDao( solrDao.getSolrClient(solrDao.getZkHosts()), - solrDao.getSolrSearchDao()); + solrDao.getSolrSearchDao(), config); SolrMetaAlertRetrieveLatestDao retrieveLatestDao = new SolrMetaAlertRetrieveLatestDao(solrDao); SolrMetaAlertUpdateDao updateDao = new SolrMetaAlertUpdateDao(solrDao, searchDao, retrieveLatestDao, config);