Merge branch 'master' into feature/METRON-1416-upgrade-solr
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/4c4452e5 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/4c4452e5 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/4c4452e5 Branch: refs/heads/feature/METRON-1554-pcap-query-panel Commit: 4c4452e59c93306a049e6879f7b6fed2ee86c74e Parents: 4768561 Author: cstella <ceste...@gmail.com> Authored: Tue Jun 12 21:40:01 2018 -0400 Committer: cstella <ceste...@gmail.com> Committed: Tue Jun 12 21:40:01 2018 -0400 ---------------------------------------------------------------------- .../dao/ElasticsearchSearchDao.java | 30 +-- .../ElasticsearchMetaAlertIntegrationTest.java | 223 ++++++++++--------- 2 files changed, 134 insertions(+), 119 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/4c4452e5/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java index 5725534..5cd0a4d 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java @@ -443,22 +443,24 @@ public class ElasticsearchSearchDao implements SearchDao { private List<GroupResult> getGroupResults(GroupRequest groupRequest, int index, Aggregations aggregations, Map<String, FieldType> commonColumnMetadata) { List<Group> groups = groupRequest.getGroups(); String field = groups.get(index).getField(); - Terms terms = aggregations.get(getGroupByAggregationName(field)); List<GroupResult> searchResultGroups = new ArrayList<>(); - for(Bucket bucket: terms.getBuckets()) { - GroupResult groupResult = new GroupResult(); - groupResult.setKey(formatKey(bucket.getKey(), commonColumnMetadata.get(field))); - groupResult.setTotal(bucket.getDocCount()); - Optional<String> scoreField = groupRequest.getScoreField(); - if (scoreField.isPresent()) { - Sum score = bucket.getAggregations().get(getSumAggregationName(scoreField.get())); - groupResult.setScore(score.getValue()); - } - if (index < groups.size() - 1) { - groupResult.setGroupedBy(groups.get(index + 1).getField()); - groupResult.setGroupResults(getGroupResults(groupRequest, index + 1, bucket.getAggregations(), commonColumnMetadata)); + if(aggregations != null) { + Terms terms = aggregations.get(getGroupByAggregationName(field)); + for (Bucket bucket : terms.getBuckets()) { + GroupResult groupResult = new GroupResult(); + groupResult.setKey(formatKey(bucket.getKey(), commonColumnMetadata.get(field))); + groupResult.setTotal(bucket.getDocCount()); + Optional<String> scoreField = groupRequest.getScoreField(); + if (scoreField.isPresent()) { + Sum score = bucket.getAggregations().get(getSumAggregationName(scoreField.get())); + groupResult.setScore(score.getValue()); + } + if (index < groups.size() - 1) { + groupResult.setGroupedBy(groups.get(index + 1).getField()); + groupResult.setGroupResults(getGroupResults(groupRequest, index + 1, bucket.getAggregations(), commonColumnMetadata)); + } + searchResultGroups.add(groupResult); } - searchResultGroups.add(groupResult); } return searchResultGroups; } http://git-wip-us.apache.org/repos/asf/metron/blob/4c4452e5/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java index 99f0490..224ec56 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java @@ -1,20 +1,21 @@ + /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ package org.apache.metron.elasticsearch.integration; @@ -30,12 +31,14 @@ import java.io.IOException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Function; import java.util.stream.Collectors; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.Constants; @@ -43,6 +46,7 @@ import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.elasticsearch.dao.ElasticsearchDao; import org.apache.metron.elasticsearch.dao.ElasticsearchMetaAlertDao; import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent; +import org.apache.metron.guava.collect.ImmutableList; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.IndexDao; import org.apache.metron.indexing.dao.metaalert.MetaAlertDao; @@ -58,28 +62,44 @@ import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +@RunWith(Parameterized.class) public class ElasticsearchMetaAlertIntegrationTest extends MetaAlertIntegrationTest { private static IndexDao esDao; private static ElasticSearchComponent es; protected static final String INDEX_DIR = "target/elasticsearch_meta"; + private static String POSTFIX= new SimpleDateFormat(DATE_FORMAT).format(new Date()); + private static final String INDEX_RAW = SENSOR_NAME + POSTFIX; + protected static final String INDEX = INDEX_RAW + "_index"; + protected List<String> queryIndices = null; + + @Parameterized.Parameters + public static Collection<Object[]> data() { + Function<List<String>, List<String>> asteriskTransform = x -> ImmutableList.of("*"); + Function<List<String>, List<String>> explicitTransform = + allIndices -> allIndices.stream().map(x -> x.replace("_index", "")) + .collect(Collectors.toCollection(ArrayList::new)); + return Arrays.asList(new Object[][]{ + {asteriskTransform}, + {explicitTransform} + } + ); + } - protected static final String INDEX = - SENSOR_NAME + "_" + new SimpleDateFormat(DATE_FORMAT).format(new Date()); - protected static final String INDEX_WITH_SEPARATOR = INDEX + "_index"; + public ElasticsearchMetaAlertIntegrationTest(Function<List<String>, List<String>> queryIndices) { + this.queryIndices = queryIndices.apply(allIndices); + } - protected ArrayList<String> queryIndices = allIndices.stream().map(x -> x.replace("_index", "")) - .collect(Collectors.toCollection(ArrayList::new)); /** { - "properties": { - "alert": { - "type": "nested" - } - } + "properties": { + "metron_alert": { "type": "nested" } + } } */ @Multiline @@ -87,22 +107,15 @@ public class ElasticsearchMetaAlertIntegrationTest extends MetaAlertIntegrationT /** * { - "%MAPPING_NAME%_doc" : { - "properties" : { - "guid" : { - "type" : "keyword" - }, - "ip_src_addr" : { - "type" : "keyword" - }, - "score" : { - "type" : "integer" - }, - "alert" : { - "type" : "nested" - } - } - } + "%MAPPING_NAME%_doc" : { + "properties" : { + "guid" : { "type" : "keyword" }, + "ip_src_addr" : { "type" : "keyword" }, + "score" : { "type" : "integer" }, + "metron_alert" : { "type" : "nested" }, + "source:type" : { "type" : "keyword"} + } + } } */ @Multiline @@ -115,16 +128,16 @@ public class ElasticsearchMetaAlertIntegrationTest extends MetaAlertIntegrationT // setup the client es = new ElasticSearchComponent.Builder() - .withHttpPort(9211) - .withIndexDir(new File(INDEX_DIR)) - .build(); + .withHttpPort(9211) + .withIndexDir(new File(INDEX_DIR)) + .build(); es.start(); } @Before public void setup() throws IOException { - es.createIndexWithMapping(METAALERTS_INDEX, METAALERT_DOC, template.replace("%MAPPING_NAME%", "metaalert")); - es.createIndexWithMapping(INDEX, "index_doc", template.replace("%MAPPING_NAME%", "index")); + es.createIndexWithMapping(METAALERTS_INDEX, METAALERT_DOC, template.replace("%MAPPING_NAME%", METAALERT_TYPE)); + es.createIndexWithMapping(INDEX, "test_doc", template.replace("%MAPPING_NAME%", "test")); AccessConfig accessConfig = new AccessConfig(); Map<String, Object> globalConfig = new HashMap<String, Object>() { @@ -173,34 +186,34 @@ public class ElasticsearchMetaAlertIntegrationTest extends MetaAlertIntegrationT alerts.get(2).put("ip_src_port", 8008); alerts.get(3).put("ip_src_addr", "192.168.1.4"); alerts.get(3).put("ip_src_port", 8007); - addRecords(alerts, INDEX_WITH_SEPARATOR, SENSOR_NAME); + addRecords(alerts, INDEX, SENSOR_NAME); // Put the nested type into the test index, so that it'll match appropriately setupTypings(); // Load metaAlerts Map<String, Object> activeMetaAlert = buildMetaAlert("meta_active", MetaAlertStatus.ACTIVE, - Optional.of(Arrays.asList(alerts.get(0), alerts.get(1)))); + Optional.of(Arrays.asList(alerts.get(0), alerts.get(1)))); Map<String, Object> inactiveMetaAlert = buildMetaAlert("meta_inactive", - MetaAlertStatus.INACTIVE, - Optional.of(Arrays.asList(alerts.get(2), alerts.get(3)))); + MetaAlertStatus.INACTIVE, + Optional.of(Arrays.asList(alerts.get(2), alerts.get(3)))); // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically. addRecords(Arrays.asList(activeMetaAlert, inactiveMetaAlert), METAALERTS_INDEX, - METAALERT_TYPE); + METAALERT_TYPE); // Verify load was successful findCreatedDocs(Arrays.asList( - new GetRequest("message_0", SENSOR_NAME), - new GetRequest("message_1", SENSOR_NAME), - new GetRequest("message_2", SENSOR_NAME), - new GetRequest("message_3", SENSOR_NAME), - new GetRequest("meta_active", METAALERT_TYPE), - new GetRequest("meta_inactive", METAALERT_TYPE))); + new GetRequest("message_0", SENSOR_NAME), + new GetRequest("message_1", SENSOR_NAME), + new GetRequest("message_2", SENSOR_NAME), + new GetRequest("message_3", SENSOR_NAME), + new GetRequest("meta_active", METAALERT_TYPE), + new GetRequest("meta_inactive", METAALERT_TYPE))); SearchResponse searchResponse = metaDao.search(new SearchRequest() { { setQuery( - "(ip_src_addr:192.168.1.1 AND ip_src_port:8009) OR (alert.ip_src_addr:192.168.1.1 AND alert.ip_src_port:8009)"); + "(ip_src_addr:192.168.1.1 AND ip_src_port:8009) OR (metron_alert.ip_src_addr:192.168.1.1 AND metron_alert.ip_src_port:8009)"); setIndices(Collections.singletonList(METAALERT_TYPE)); setFrom(0); setSize(5); @@ -219,8 +232,8 @@ public class ElasticsearchMetaAlertIntegrationTest extends MetaAlertIntegrationT searchResponse = metaDao.search(new SearchRequest() { { setQuery( - "(ip_src_addr:192.168.1.1 AND ip_src_port:8010)" - + " OR (alert.ip_src_addr:192.168.1.1 AND alert.ip_src_port:8010)"); + "(ip_src_addr:192.168.1.1 AND ip_src_port:8010)" + + " OR (metron_alert.ip_src_addr:192.168.1.1 AND metron_alert.ip_src_port:8010)"); setIndices(queryIndices); setFrom(0); setSize(5); @@ -235,16 +248,16 @@ public class ElasticsearchMetaAlertIntegrationTest extends MetaAlertIntegrationT // Nested query should match a nested alert Assert.assertEquals(1, searchResponse.getTotal()); Assert.assertEquals("meta_active", - searchResponse.getResults().get(0).getSource().get("guid")); + searchResponse.getResults().get(0).getSource().get("guid")); // Query against all indices. The child alert has no actual attached meta alerts, and should // be returned on its own. searchResponse = metaDao.search(new SearchRequest() { { setQuery( - "(ip_src_addr:192.168.1.3 AND ip_src_port:8008)" - + " OR (alert.ip_src_addr:192.168.1.3 AND alert.ip_src_port:8008)"); - setIndices(queryIndices); + "(ip_src_addr:192.168.1.3 AND ip_src_port:8008)" + + " OR (metron_alert.ip_src_addr:192.168.1.3 AND metron_alert.ip_src_port:8008)"); + setIndices(Collections.singletonList("*")); setFrom(0); setSize(1); setSort(Collections.singletonList(new SortField() { @@ -258,83 +271,83 @@ public class ElasticsearchMetaAlertIntegrationTest extends MetaAlertIntegrationT // Nested query should match a plain alert Assert.assertEquals(1, searchResponse.getTotal()); Assert.assertEquals("message_2", - searchResponse.getResults().get(0).getSource().get("guid")); + searchResponse.getResults().get(0).getSource().get("guid")); } @Override protected long getMatchingAlertCount(String fieldName, Object fieldValue) - throws IOException, InterruptedException { + throws IOException, InterruptedException { long cnt = 0; for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t, Thread.sleep(SLEEP_MS)) { List<Map<String, Object>> docs = es - .getAllIndexedDocs(INDEX_WITH_SEPARATOR, SENSOR_NAME + "_doc"); + .getAllIndexedDocs(INDEX, SENSOR_NAME + "_doc"); cnt = docs - .stream() - .filter(d -> { - Object newfield = d.get(fieldName); - return newfield != null && newfield.equals(fieldValue); - }).count(); + .stream() + .filter(d -> { + Object newfield = d.get(fieldName); + return newfield != null && newfield.equals(fieldValue); + }).count(); } return cnt; } @Override protected long getMatchingMetaAlertCount(String fieldName, String fieldValue) - throws IOException, InterruptedException { + throws IOException, InterruptedException { long cnt = 0; for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t, Thread.sleep(SLEEP_MS)) { List<Map<String, Object>> docs = es - .getAllIndexedDocs(METAALERTS_INDEX, METAALERT_DOC); + .getAllIndexedDocs(METAALERTS_INDEX, METAALERT_DOC); cnt = docs - .stream() - .filter(d -> { - @SuppressWarnings("unchecked") - List<Map<String, Object>> alerts = (List<Map<String, Object>>) d - .get(ALERT_FIELD); - - for (Map<String, Object> alert : alerts) { - Object newField = alert.get(fieldName); - if (newField != null && newField.equals(fieldValue)) { - return true; - } - } - - return false; - }).count(); + .stream() + .filter(d -> { + @SuppressWarnings("unchecked") + List<Map<String, Object>> alerts = (List<Map<String, Object>>) d + .get(ALERT_FIELD); + + for (Map<String, Object> alert : alerts) { + Object newField = alert.get(fieldName); + if (newField != null && newField.equals(fieldValue)) { + return true; + } + } + + return false; + }).count(); } return cnt; } @Override protected void addRecords(List<Map<String, Object>> inputData, String index, String docType) - throws IOException { + throws IOException { es.add(index, docType, inputData.stream().map(m -> { - try { - return JSONUtils.INSTANCE.toJSON(m, true); - } catch (JsonProcessingException e) { - throw new IllegalStateException(e.getMessage(), e); - } - } - ).collect(Collectors.toList()) + try { + return JSONUtils.INSTANCE.toJSON(m, true); + } catch (JsonProcessingException e) { + throw new IllegalStateException(e.getMessage(), e); + } + } + ).collect(Collectors.toList()) ); } @Override protected void setupTypings() { - ((ElasticsearchDao) esDao).getClient().admin().indices().preparePutMapping(INDEX_WITH_SEPARATOR) - .setType("test_doc") - .setSource(nestedAlertMapping) - .get(); + ((ElasticsearchDao) esDao).getClient().admin().indices().preparePutMapping(INDEX) + .setType("test_doc") + .setSource(nestedAlertMapping) + .get(); } @Override protected String getTestIndexName() { - return INDEX; + return INDEX_RAW; } @Override protected String getTestIndexFullName() { - return INDEX_WITH_SEPARATOR; + return INDEX; } @Override @@ -361,4 +374,4 @@ public class ElasticsearchMetaAlertIntegrationTest extends MetaAlertIntegrationT protected boolean isEmptyMetaAlertList() { return true; } -} +} \ No newline at end of file