http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java index 4c03526..1efcc39 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java @@ -17,6 +17,7 @@ */ package org.apache.metron.elasticsearch.integration; +import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.interfaces.FieldNameConverter; import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent; import org.apache.metron.elasticsearch.writer.ElasticsearchFieldNameConverter; @@ -35,6 +36,7 @@ import java.util.Date; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; public class ElasticsearchIndexingIntegrationTest extends IndexingIntegrationTest { @@ -42,6 +44,20 @@ public class ElasticsearchIndexingIntegrationTest extends IndexingIntegrationTes private String dateFormat = "yyyy.MM.dd.HH"; private String index = "yaf_index_" + new SimpleDateFormat(dateFormat).format(new Date()); private FieldNameConverter fieldNameConverter = new ElasticsearchFieldNameConverter(); + /** + * { + * "yaf_doc": { + * "properties": { + * "source:type": { "type": "keyword" }, + * "guid": { "type": "keyword" }, + * "isn": { "type": "text" } + * } + * } + * } + */ + @Multiline + private static String mapping; + @Override public FieldNameConverter getFieldNameConverter() { @@ -53,6 +69,7 @@ public class ElasticsearchIndexingIntegrationTest extends IndexingIntegrationTes return new ElasticSearchComponent.Builder() .withHttpPort(9211) .withIndexDir(new File(indexDir)) + .withMapping(index, "yaf_doc", mapping) .build(); } @@ -61,6 +78,7 @@ public class ElasticsearchIndexingIntegrationTest extends IndexingIntegrationTes return new Processor<List<Map<String, Object>>>() { List<Map<String, Object>> docs = null; List<byte[]> errors = null; + final AtomicInteger missCount = new AtomicInteger(0); @Override public ReadinessState process(ComponentRunner runner) { ElasticSearchComponent elasticSearchComponent = runner.getComponent("search", ElasticSearchComponent.class); @@ -70,7 +88,9 @@ public class ElasticsearchIndexingIntegrationTest extends IndexingIntegrationTes try { docs = elasticSearchComponent.getAllIndexedDocs(index, testSensorType + "_doc"); docsFromDisk = readDocsFromDisk(hdfsDir); - System.out.println(docs.size() + " vs " + inputMessages.size() + " vs " + docsFromDisk.size()); + if(missCount.incrementAndGet() >= NUM_RETRIES/2) { + System.out.println(missCount.get() + ": " + docs.size() + " vs " + inputMessages.size() + " vs " + docsFromDisk.size()); + } } catch (IOException e) { throw new IllegalStateException("Unable to retrieve indexed documents.", e); } @@ -84,7 +104,6 @@ public class ElasticsearchIndexingIntegrationTest extends IndexingIntegrationTes return ReadinessState.READY; } } else { - System.out.println("Missed index..."); return ReadinessState.NOT_READY; } }
http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java index 35f40ca..9e74fb6 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java @@ -23,9 +23,10 @@ import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERTS_INDEX; import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERT_FIELD; import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERT_TYPE; import static org.apache.metron.indexing.dao.MetaAlertDao.STATUS_FIELD; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.base.Joiner; +import com.google.common.collect.Iterables; import java.io.File; import java.io.IOException; import java.text.SimpleDateFormat; @@ -34,9 +35,11 @@ import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.Constants; @@ -159,6 +162,29 @@ public class ElasticsearchMetaAlertIntegrationTest { @Multiline public static String statusPatchRequest; + /** + * { + "%MAPPING_NAME%_doc" : { + "properties" : { + "guid" : { + "type" : "keyword" + }, + "ip_src_addr" : { + "type" : "keyword" + }, + "score" : { + "type" : "integer" + }, + "alert" : { + "type" : "nested" + } + } + } + } + */ + @Multiline + public static String template; + @BeforeClass public static void setupBefore() throws Exception { // setup the client @@ -188,8 +214,8 @@ public class ElasticsearchMetaAlertIntegrationTest { @Before public void setup() throws IOException { - es.createIndexWithMapping(METAALERTS_INDEX, MetaAlertDao.METAALERT_DOC, - buildMetaMappingSource()); + es.createIndexWithMapping(METAALERTS_INDEX, MetaAlertDao.METAALERT_DOC, template.replace("%MAPPING_NAME%", "metaalert")); + es.createIndexWithMapping(INDEX, "index_doc", template.replace("%MAPPING_NAME%", "index")); } @AfterClass @@ -204,27 +230,6 @@ public class ElasticsearchMetaAlertIntegrationTest { es.reset(); } - protected static String buildMetaMappingSource() throws IOException { - return jsonBuilder().prettyPrint() - .startObject() - .startObject(MetaAlertDao.METAALERT_DOC) - .startObject("properties") - .startObject("guid") - .field("type", "string") - .field("index", "not_analyzed") - .endObject() - .startObject("score") - .field("type", "integer") - .field("index", "not_analyzed") - .endObject() - .startObject("alert") - .field("type", "nested") - .endObject() - .endObject() - .endObject() - .endObject() - .string(); - } @Test public void shouldGetAllMetaAlertsForAlert() throws Exception { @@ -259,7 +264,11 @@ public class ElasticsearchMetaAlertIntegrationTest { SearchResponse searchResponse0 = metaDao.getAllMetaAlertsForAlert("message_0"); List<SearchResult> searchResults0 = searchResponse0.getResults(); Assert.assertEquals(13, searchResults0.size()); - Assert.assertEquals(metaAlerts.get(0), searchResults0.get(0).getSource()); + Set<Map<String, Object>> resultSet = new HashSet<>(); + Iterables.addAll(resultSet, Iterables.transform(searchResults0, r -> r.getSource())); + StringBuffer reason = new StringBuffer("Unable to find " + metaAlerts.get(0) + "\n"); + reason.append(Joiner.on("\n").join(resultSet)); + Assert.assertTrue(reason.toString(), resultSet.contains(metaAlerts.get(0))); // Verify no meta alerts are returned because message_1 was not added to any SearchResponse searchResponse1 = metaDao.getAllMetaAlertsForAlert("message_1"); @@ -760,14 +769,14 @@ public class ElasticsearchMetaAlertIntegrationTest { // Query against all indices. The child alert has no actual attached meta alerts, and should // be returned on its own. - searchResponse = metaDao.search(new SearchRequest() { + searchResponse = metaDao.search(new SearchRequest() { { setQuery( "(ip_src_addr:192.168.1.3 AND ip_src_port:8008)" + " OR (alert.ip_src_addr:192.168.1.3 AND alert.ip_src_port:8008)"); setIndices(Collections.singletonList("*")); setFrom(0); - setSize(5); + setSize(1); setSort(Collections.singletonList(new SortField() { { setField(Constants.GUID); http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java index eac4851..3949c6d 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java @@ -17,17 +17,24 @@ */ package org.apache.metron.elasticsearch.integration; + +import java.io.File; +import java.util.HashMap; +import java.io.IOException; +import java.util.concurrent.ExecutionException; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.elasticsearch.dao.ElasticsearchDao; import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.IndexDao; +import org.apache.metron.indexing.dao.MetaAlertDao; import org.apache.metron.indexing.dao.SearchIntegrationTest; import org.apache.metron.integration.InMemoryComponent; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; @@ -53,8 +60,11 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { * "bro_doc": { * "properties": { * "source:type": { - * "type": "string", - * "index": "not_analyzed" + * "type": "text", + * "fielddata" : "true" + * }, + * "guid" : { + * "type" : "keyword" * }, * "ip_src_addr": { * "type": "ip" @@ -82,10 +92,12 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { * "type": "geo_point" * }, * "bro_field": { - * "type": "string" + * "type": "text", + * "fielddata" : "true" * }, * "duplicate_name_field": { - * "type": "string" + * "type": "text", + * "fielddata" : "true" * }, * "alert": { * "type": "nested" @@ -102,8 +114,11 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { * "snort_doc": { * "properties": { * "source:type": { - * "type": "string", - * "index": "not_analyzed" + * "type": "text", + * "fielddata" : "true" + * }, + * "guid" : { + * "type" : "keyword" * }, * "ip_src_addr": { * "type": "ip" @@ -170,8 +185,23 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { * { * "metaalert_doc": { * "properties": { - * "source:type": { "type": "string" }, - * "alert": { "type": "nested"} + * "guid": { "type": "keyword" }, + * "alert": { + * "type": "nested", + * "properties": { + * "guid": { "type": "keyword" } + * } + * }, + * "average": { "type": "keyword" }, + * "min" : { "type": "keyword" }, + * "median" : { "type": "keyword" }, + * "max": { "type": "keyword" }, + * "count": { "type": "keyword" }, + * "sum": { "type": "keyword" }, + * "source:type": { + * "type": "text", + * "fielddata" : "true" + * } * } * } * } @@ -216,10 +246,10 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { .addMapping("bro_doc", broTypeMappings).addMapping("bro_doc_default", broDefaultStringMappings).get(); es.getClient().admin().indices().prepareCreate("snort_index_2017.01.01.02") .addMapping("snort_doc", snortTypeMappings).get(); - es.getClient().admin().indices().prepareCreate("metaalert_index") - .addMapping("metaalert_doc", metaAlertTypeMappings).get(); + es.getClient().admin().indices().prepareCreate(MetaAlertDao.METAALERTS_INDEX) + .addMapping(MetaAlertDao.METAALERT_DOC, metaAlertTypeMappings).get(); - BulkRequestBuilder bulkRequest = es.getClient().prepareBulk().setRefresh(true); + BulkRequestBuilder bulkRequest = es.getClient().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); JSONArray broArray = (JSONArray) new JSONParser().parse(broData); for(Object o: broArray) { JSONObject jsonObject = (JSONObject) o; http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java index 3ef9379..f95ef21 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java @@ -17,7 +17,18 @@ */ package org.apache.metron.elasticsearch.integration.components; +import static java.util.Arrays.asList; + import com.fasterxml.jackson.core.type.TypeReference; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; import org.apache.commons.io.FileUtils; import org.apache.metron.common.Constants; import org.apache.metron.common.utils.JSONUtils; @@ -29,210 +40,251 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.node.InternalSettingsPreparer; import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeBuilder; +import org.elasticsearch.node.NodeValidationException; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchHit; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; +import org.elasticsearch.transport.Netty4Plugin; public class ElasticSearchComponent implements InMemoryComponent { - public static class Builder{ - private int httpPort; - private File indexDir; - private Map<String, String> extraElasticSearchSettings = null; - public Builder withHttpPort(int httpPort) { - this.httpPort = httpPort; - return this; - } - public Builder withIndexDir(File indexDir) { - this.indexDir = indexDir; - return this; - } - public Builder withExtraElasticSearchSettings(Map<String, String> extraElasticSearchSettings) { - this.extraElasticSearchSettings = extraElasticSearchSettings; - return this; - } - public ElasticSearchComponent build() { - return new ElasticSearchComponent(httpPort, indexDir, extraElasticSearchSettings); - } - } - - private Client client; - private Node node; + private static class Mapping { + String index; + String docType; + String mapping; + + public Mapping(String index, String docType, String mapping) { + this.index = index; + this.docType = docType; + this.mapping = mapping; + } + } + + public static class Builder { + private int httpPort; private File indexDir; - private Map<String, String> extraElasticSearchSettings; + private Map<String, String> extraElasticSearchSettings = null; + private List<Mapping> mappings = new ArrayList<>(); - public ElasticSearchComponent(int httpPort, File indexDir) { - this(httpPort, indexDir, null); + public Builder withMapping(String index, String docType, String mapping) { + mappings.add(new Mapping(index, docType, mapping)); + return this; } - public ElasticSearchComponent(int httpPort, File indexDir, Map<String, String> extraElasticSearchSettings) { - this.httpPort = httpPort; - this.indexDir = indexDir; - this.extraElasticSearchSettings = extraElasticSearchSettings; + + public Builder withHttpPort(int httpPort) { + this.httpPort = httpPort; + return this; } - public Client getClient() { - return client; + + public Builder withIndexDir(File indexDir) { + this.indexDir = indexDir; + return this; } - private void cleanDir(File dir) throws IOException { - if(dir.exists()) { - FileUtils.deleteDirectory(dir); - } - dir.mkdirs(); + public Builder withExtraElasticSearchSettings( + Map<String, String> extraElasticSearchSettings) { + this.extraElasticSearchSettings = extraElasticSearchSettings; + return this; } - public BulkResponse add(String indexName, String sensorType, String... docs) throws IOException { - List<String> d = new ArrayList<>(); - Collections.addAll(d, docs); - return add(indexName, sensorType, d); + public ElasticSearchComponent build() { + return new ElasticSearchComponent(httpPort, indexDir, extraElasticSearchSettings, mappings); } + } - public BulkResponse add(String indexName, String sensorType, Iterable<String> docs) throws IOException { - BulkRequestBuilder bulkRequest = getClient().prepareBulk(); - for(String doc : docs) { - IndexRequestBuilder indexRequestBuilder = getClient().prepareIndex(indexName, - sensorType + "_doc"); + private static final String STARTUP_TIMEOUT = "60s"; + private Client client; + private Node node; + private int httpPort; + private File indexDir; + private Map<String, String> extraElasticSearchSettings; + private List<Mapping> mappings; - indexRequestBuilder = indexRequestBuilder.setSource(doc); - Map<String, Object> esDoc = JSONUtils.INSTANCE.load(doc, new TypeReference<Map<String, Object>>() { - }); - indexRequestBuilder.setId((String) esDoc.get(Constants.GUID)); - Object ts = esDoc.get("timestamp"); - if(ts != null) { - indexRequestBuilder = indexRequestBuilder.setTimestamp(ts.toString()); - } - bulkRequest.add(indexRequestBuilder); - } + public ElasticSearchComponent(int httpPort, File indexDir, + Map<String, String> extraElasticSearchSettings, List<Mapping> mappings) { + this.httpPort = httpPort; + this.indexDir = indexDir; + this.extraElasticSearchSettings = extraElasticSearchSettings; + this.mappings = mappings; + } - BulkResponse response = bulkRequest.execute().actionGet(); - if(response.hasFailures()) { - throw new IOException(response.buildFailureMessage()); - } - return response; + @Override + public void start() throws UnableToStartException { + File logDir = new File(indexDir, "/logs"); + File dataDir = new File(indexDir, "/data"); + try { + cleanDir(logDir); + cleanDir(dataDir); + } catch (IOException e) { + throw new UnableToStartException("Unable to clean log or data directories", e); } - public void createIndexWithMapping(String indexName, String mappingType, String mappingSource) - throws IOException { - CreateIndexResponse cir = client.admin().indices().prepareCreate(indexName) - .addMapping(mappingType, mappingSource) - .get(); + Settings.Builder settingsBuilder = Settings.builder() + .put("cluster.name", "metron") + .put("path.logs",logDir.getAbsolutePath()) + .put("path.data",dataDir.getAbsolutePath()) + .put("path.home", indexDir.getAbsoluteFile()) + .put("transport.type", "netty4") + .put("http.enabled", "false"); - if (!cir.isAcknowledged()) { - throw new IOException("Create index was not acknowledged"); - } + if (extraElasticSearchSettings != null) { + settingsBuilder = settingsBuilder.put(extraElasticSearchSettings); } - @Override - public void start() throws UnableToStartException { - File logDir= new File(indexDir, "/logs"); - File dataDir= new File(indexDir, "/data"); - try { - cleanDir(logDir); - cleanDir(dataDir); - - } catch (IOException e) { - throw new UnableToStartException("Unable to clean log or data directories", e); - } - - Settings.Builder settingsBuilder = Settings.settingsBuilder() - .put("node.http.enabled", true) - .put("http.port", httpPort) - .put("path.logs",logDir.getAbsolutePath()) - .put("path.data",dataDir.getAbsolutePath()) - .put("path.home", indexDir.getAbsoluteFile()) - .put("index.number_of_shards", 1) - .put("node.mode", "network") - .put("index.number_of_replicas", 1); - - if(extraElasticSearchSettings != null) { - - settingsBuilder = settingsBuilder.put(extraElasticSearchSettings); - - } - - node = NodeBuilder.nodeBuilder().settings(settingsBuilder).clusterName("metron").node(); - node.start(); - - client = node.client(); - - waitForCluster(client, ClusterHealthStatus.YELLOW, new TimeValue(60000)); - - } - - public static void waitForCluster(ElasticsearchClient client, ClusterHealthStatus status, TimeValue timeout) throws UnableToStartException { - try { - ClusterHealthResponse healthResponse = - (ClusterHealthResponse)client.execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest().waitForStatus(status).timeout(timeout)).actionGet(); - if (healthResponse != null && healthResponse.isTimedOut()) { - throw new UnableToStartException("cluster state is " + healthResponse.getStatus().name() - + " and not " + status.name() - + ", from here on, everything will fail!"); - } - } catch (ElasticsearchTimeoutException e) { - throw new UnableToStartException("timeout, cluster does not respond to health request, cowardly refusing to continue with operations"); - } - } - - public List<Map<String, Object>> getAllIndexedDocs(String index, String sourceType) throws IOException { - return getAllIndexedDocs(index, sourceType, null); - } - public List<Map<String, Object>> getAllIndexedDocs(String index, String sourceType, String subMessage) throws IOException { - getClient().admin().indices().refresh(new RefreshRequest()); - SearchResponse response = getClient().prepareSearch(index) - .setTypes(sourceType) - .setSource("message") - .setFrom(0) - .setSize(1000) - .execute().actionGet(); - List<Map<String, Object>> ret = new ArrayList<Map<String, Object>>(); - for (SearchHit hit : response.getHits()) { - Object o = null; - if(subMessage == null) { - o = hit.getSource(); - } - else { - o = hit.getSource().get(subMessage); - } - ret.add((Map<String, Object>)(o)); - } - return ret; - } - public boolean hasIndex(String indexName) { - Set<String> indices = getClient().admin() - .indices() - .stats(new IndicesStatsRequest()) - .actionGet() - .getIndices() - .keySet(); - return indices.contains(indexName); + node = new TestNode(settingsBuilder.build(), asList(Netty4Plugin.class)); + client = node.client(); + try { + node.start(); + } catch (NodeValidationException e) { + throw new UnableToStartException("Error starting ES node.", e); + } + waitForCluster(client, ClusterHealthStatus.YELLOW, STARTUP_TIMEOUT); + for(Mapping m : Optional.ofNullable(mappings).orElse(new ArrayList<>())) { + client.admin().indices().prepareCreate(m.index) + .addMapping(m.docType, m.mapping).get(); + } + } + + private void cleanDir(File dir) throws IOException { + if (dir.exists()) { + FileUtils.deleteDirectory(dir); + } + dir.mkdirs(); + } + + // ES 5.x+ needs this to startup a node without using their test framework + private static class TestNode extends Node { + + private TestNode(Settings preparedSettings, + Collection<Class<? extends Plugin>> classpathPlugins) { + super(InternalSettingsPreparer.prepareEnvironment(preparedSettings, null), classpathPlugins); + } + + } + + public static void waitForCluster(Client client, ClusterHealthStatus statusThreshold, + String timeout) throws UnableToStartException { + try { + ClusterHealthResponse healthResponse = (ClusterHealthResponse) client + .execute(ClusterHealthAction.INSTANCE, + new ClusterHealthRequest().waitForStatus(statusThreshold).timeout(timeout)) + .actionGet(); + if (healthResponse != null && healthResponse.isTimedOut()) { + throw new UnableToStartException("cluster state is " + healthResponse.getStatus().name() + + " and not " + statusThreshold.name() + + ", from here on, everything will fail!"); + } + } catch (ElasticsearchTimeoutException e) { + throw new UnableToStartException( + "timeout, cluster does not respond to health request, cowardly refusing to continue with operations"); + } + } + public Client getClient() { + return client; + } + + public BulkResponse add(String indexName, String sensorType, String... docs) throws IOException { + List<String> d = new ArrayList<>(); + Collections.addAll(d, docs); + return add(indexName, sensorType, d); + } + + public BulkResponse add(String indexName, String sensorType, Iterable<String> docs) + throws IOException { + BulkRequestBuilder bulkRequest = getClient().prepareBulk(); + for (String doc : docs) { + IndexRequestBuilder indexRequestBuilder = getClient() + .prepareIndex(indexName, sensorType + "_doc"); + + indexRequestBuilder = indexRequestBuilder.setSource(doc); + Map<String, Object> esDoc = JSONUtils.INSTANCE + .load(doc, new TypeReference<Map<String, Object>>() { + }); + indexRequestBuilder.setId((String) esDoc.get(Constants.GUID)); + Object ts = esDoc.get("timestamp"); + if (ts != null) { + indexRequestBuilder = indexRequestBuilder.setTimestamp(ts.toString()); + } + bulkRequest.add(indexRequestBuilder); } + BulkResponse response = bulkRequest.execute().actionGet(); + if (response.hasFailures()) { + throw new IOException(response.buildFailureMessage()); + } + return response; + } + + public void createIndexWithMapping(String indexName, String mappingType, String mappingSource) + throws IOException { + CreateIndexResponse cir = client.admin().indices().prepareCreate(indexName) + .addMapping(mappingType, mappingSource) + .get(); + + if (!cir.isAcknowledged()) { + throw new IOException("Create index was not acknowledged"); + } + } + + public List<Map<String, Object>> getAllIndexedDocs(String index, String sourceType) + throws IOException { + return getAllIndexedDocs(index, sourceType, null); + } + + public List<Map<String, Object>> getAllIndexedDocs(String index, String sourceType, + String subMessage) throws IOException { + getClient().admin().indices().refresh(new RefreshRequest()); + SearchResponse response = getClient().prepareSearch(index) + .setTypes(sourceType) +// .setSource("message") ?? + .setFrom(0) + .setSize(1000) + .execute().actionGet(); + List<Map<String, Object>> ret = new ArrayList<Map<String, Object>>(); + for (SearchHit hit : response.getHits()) { + Object o = null; + if (subMessage == null) { + o = hit.getSource(); + } else { + o = hit.getSource().get(subMessage); + } + ret.add((Map<String, Object>) (o)); + } + return ret; + } + + public boolean hasIndex(String indexName) { + Set<String> indices = getClient().admin() + .indices() + .stats(new IndicesStatsRequest()) + .actionGet() + .getIndices() + .keySet(); + return indices.contains(indexName); + + } + @Override public void stop() { + try { node.close(); - node = null; - client = null; + } catch (IOException e) { + throw new RuntimeException("Unable to stop node." , e); + } + node = null; + client = null; } @Override http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-platform/metron-enrichment/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/pom.xml b/metron-platform/metron-enrichment/pom.xml index aab1826..e82b86b 100644 --- a/metron-platform/metron-enrichment/pom.xml +++ b/metron-platform/metron-enrichment/pom.xml @@ -200,6 +200,14 @@ <scope>provided</scope> <exclusions> <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + </exclusion> + <exclusion> <artifactId>servlet-api</artifactId> <groupId>javax.servlet</groupId> </exclusion> @@ -213,7 +221,18 @@ </exclusion> </exclusions> </dependency> - + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-api</artifactId> + <version>${global_log4j_core_version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + <version>${global_log4j_core_version}</version> + <scope>test</scope> + </dependency> <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-platform/metron-hbase/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-hbase/pom.xml b/metron-platform/metron-hbase/pom.xml index 98d7a36..746a610 100644 --- a/metron-platform/metron-hbase/pom.xml +++ b/metron-platform/metron-hbase/pom.xml @@ -140,6 +140,14 @@ <scope>provided</scope> <exclusions> <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + </exclusion> + <exclusion> <artifactId>servlet-api</artifactId> <groupId>javax.servlet</groupId> </exclusion> http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-platform/metron-indexing/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/pom.xml b/metron-platform/metron-indexing/pom.xml index ea88154..e9fe43e 100644 --- a/metron-platform/metron-indexing/pom.xml +++ b/metron-platform/metron-indexing/pom.xml @@ -32,6 +32,12 @@ </properties> <dependencies> <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + <version>2.8.2</version> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.apache.metron</groupId> <artifactId>metron-common</artifactId> <version>${project.parent.version}</version> @@ -93,6 +99,10 @@ <artifactId>log4j-slf4j-impl</artifactId> <groupId>org.apache.logging.log4j</groupId> </exclusion> + <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + </exclusion> </exclusions> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/FieldType.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/FieldType.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/FieldType.java index 1f00cf5..2abd997 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/FieldType.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/FieldType.java @@ -20,8 +20,10 @@ package org.apache.metron.indexing.dao.search; import com.fasterxml.jackson.annotation.JsonProperty; public enum FieldType { - @JsonProperty("string") - STRING("string"), + @JsonProperty("text") + TEXT("text"), + @JsonProperty("keyword") + KEYWORD("keyword"), @JsonProperty("ip") IP("ip"), @JsonProperty("integer") http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java index 4d3ff9b..72e632f 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java @@ -1,3 +1,4 @@ + /** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -18,6 +19,13 @@ package org.apache.metron.indexing.dao; import com.fasterxml.jackson.core.type.TypeReference; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.indexing.dao.search.FieldType; @@ -34,7 +42,9 @@ import org.apache.metron.integration.InMemoryComponent; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import java.util.ArrayList; import java.util.Arrays; @@ -439,8 +449,21 @@ public abstract class SearchIntegrationTest { } } + @Rule + public ExpectedException thrown = ExpectedException.none(); + @Test - public void test() throws Exception { + public void find_one_guid() throws Exception { + GetRequest request = JSONUtils.INSTANCE.load(findOneGuidQuery, GetRequest.class); + Optional<Map<String, Object>> response = dao.getLatestResult(request); + Assert.assertTrue(response.isPresent()); + Map<String, Object> doc = response.get(); + Assert.assertEquals("bro", doc.get("source:type")); + Assert.assertEquals(3, doc.get("timestamp")); + } + + @Test + public void all_query_returns_all_results() throws Exception { //All Query Testcase { SearchRequest request = JSONUtils.INSTANCE.load(allQuery, SearchRequest.class); @@ -450,34 +473,29 @@ public abstract class SearchIntegrationTest { Assert.assertEquals(10, results.size()); for(int i = 0;i < 5;++i) { Assert.assertEquals("snort", results.get(i).getSource().get("source:type")); - Assert.assertEquals(10-i, results.get(i).getSource().get("timestamp")); + Assert.assertEquals(10 - i, results.get(i).getSource().get("timestamp")); } - for(int i = 5;i < 10;++i) { + for (int i = 5; i < 10; ++i) { Assert.assertEquals("bro", results.get(i).getSource().get("source:type")); - Assert.assertEquals(10-i, results.get(i).getSource().get("timestamp")); + Assert.assertEquals(10 - i, results.get(i).getSource().get("timestamp")); } } - //Find One Guid Testcase - { - GetRequest request = JSONUtils.INSTANCE.load(findOneGuidQuery, GetRequest.class); - Optional<Map<String, Object>> response = dao.getLatestResult(request); - Assert.assertTrue(response.isPresent()); - Map<String, Object> doc = response.get(); - Assert.assertEquals("bro", doc.get("source:type")); - Assert.assertEquals(3, doc.get("timestamp")); - } //Get All Latest Guid Testcase { List<GetRequest> request = JSONUtils.INSTANCE.load(getAllLatestQuery, new TypeReference<List<GetRequest>>() { }); - Iterator<Document> response = dao.getAllLatest(request).iterator(); - Document bro2 = response.next(); - Assert.assertEquals("bro_1", bro2.getDocument().get("guid")); - Assert.assertEquals("bro", bro2.getDocument().get("source:type")); - Document snort2 = response.next(); - Assert.assertEquals("bro_2", snort2.getDocument().get("guid")); - Assert.assertEquals("bro", snort2.getDocument().get("source:type")); - Assert.assertFalse(response.hasNext()); + Map<String, Document> docs = new HashMap<>(); + + for(Document doc : dao.getAllLatest(request)) { + docs.put(doc.getGuid(), doc); + } + Assert.assertEquals(2, docs.size()); + Assert.assertTrue(docs.keySet().contains("bro-1")); + Assert.assertTrue(docs.keySet().contains("bro-2")); + for(Map.Entry<String, Document> kv : docs.entrySet()) { + Document d = kv.getValue(); + Assert.assertEquals("bro", d.getDocument().get("source:type")); + } } //Filter test case { @@ -679,43 +697,190 @@ public abstract class SearchIntegrationTest { SearchResponse response = dao.search(request); Assert.assertNull(response.getFacetCounts()); } - //Exceeded maximum results query - { - SearchRequest request = JSONUtils.INSTANCE.load(exceededMaxResultsQuery, SearchRequest.class); - try { - dao.search(request); - Assert.fail("Exception expected, but did not come."); - } - catch(InvalidSearchException ise) { - Assert.assertEquals("Search result size must be less than 100", ise.getMessage()); - } + } + + @Test + public void filter_query_filters_results() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(filterQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(3, response.getTotal()); + List<SearchResult> results = response.getResults(); + Assert.assertEquals("snort", results.get(0).getSource().get("source:type")); + Assert.assertEquals(9, results.get(0).getSource().get("timestamp")); + Assert.assertEquals("snort", results.get(1).getSource().get("source:type")); + Assert.assertEquals(7, results.get(1).getSource().get("timestamp")); + Assert.assertEquals("bro", results.get(2).getSource().get("source:type")); + Assert.assertEquals(1, results.get(2).getSource().get("timestamp")); + } + + @Test + public void sort_query_sorts_results_ascending() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(sortQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + List<SearchResult> results = response.getResults(); + for (int i = 8001; i < 8011; ++i) { + Assert.assertEquals(i, results.get(i - 8001).getSource().get("ip_src_port")); } - // getColumnMetadata with multiple indices - { - Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Arrays.asList("bro", "snort")); - Assert.assertEquals(15, fieldTypes.size()); - Assert.assertEquals(FieldType.STRING, fieldTypes.get("guid")); - Assert.assertEquals(FieldType.STRING, fieldTypes.get("source:type")); - Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr")); - Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port")); - Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field")); - Assert.assertEquals(FieldType.DATE, fieldTypes.get("timestamp")); - Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("latitude")); - Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score")); - Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert")); - Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point")); - Assert.assertEquals(FieldType.STRING, fieldTypes.get("bro_field")); - Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field")); - Assert.assertEquals(FieldType.OTHER, fieldTypes.get("duplicate_name_field")); - Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("threat:triage:score")); - Assert.assertEquals(FieldType.OTHER, fieldTypes.get("alert")); + } + + @Test + public void results_are_paginated() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(paginationQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + List<SearchResult> results = response.getResults(); + Assert.assertEquals(3, results.size()); + Assert.assertEquals("snort", results.get(0).getSource().get("source:type")); + Assert.assertEquals(6, results.get(0).getSource().get("timestamp")); + Assert.assertEquals("bro", results.get(1).getSource().get("source:type")); + Assert.assertEquals(5, results.get(1).getSource().get("timestamp")); + Assert.assertEquals("bro", results.get(2).getSource().get("source:type")); + Assert.assertEquals(4, results.get(2).getSource().get("timestamp")); + } + + @Test + public void returns_results_only_for_specified_indices() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(indexQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(5, response.getTotal()); + List<SearchResult> results = response.getResults(); + for (int i = 5, j = 0; i > 0; i--, j++) { + Assert.assertEquals("bro", results.get(j).getSource().get("source:type")); + Assert.assertEquals(i, results.get(j).getSource().get("timestamp")); } + } + + @Test + public void facet_query_yields_field_types() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(facetQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(12, response.getTotal()); + Map<String, Map<String, Long>> facetCounts = response.getFacetCounts(); + Assert.assertEquals(8, facetCounts.size()); + Map<String, Long> sourceTypeCounts = facetCounts.get("source:type"); + Assert.assertEquals(2, sourceTypeCounts.size()); + Assert.assertEquals(new Long(5), sourceTypeCounts.get("bro")); + Assert.assertEquals(new Long(5), sourceTypeCounts.get("snort")); + Map<String, Long> ipSrcAddrCounts = facetCounts.get("ip_src_addr"); + Assert.assertEquals(8, ipSrcAddrCounts.size()); + Assert.assertEquals(new Long(3), ipSrcAddrCounts.get("192.168.1.1")); + Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.2")); + Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.3")); + Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.4")); + Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.5")); + Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.6")); + Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.7")); + Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.8")); + Map<String, Long> ipSrcPortCounts = facetCounts.get("ip_src_port"); + Assert.assertEquals(10, ipSrcPortCounts.size()); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8001")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8002")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8003")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8004")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8005")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8006")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8007")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8008")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8009")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8010")); + Map<String, Long> longFieldCounts = facetCounts.get("long_field"); + Assert.assertEquals(2, longFieldCounts.size()); + Assert.assertEquals(new Long(8), longFieldCounts.get("10000")); + Assert.assertEquals(new Long(2), longFieldCounts.get("20000")); + Map<String, Long> timestampCounts = facetCounts.get("timestamp"); + Assert.assertEquals(10, timestampCounts.size()); + Assert.assertEquals(new Long(1), timestampCounts.get("1")); + Assert.assertEquals(new Long(1), timestampCounts.get("2")); + Assert.assertEquals(new Long(1), timestampCounts.get("3")); + Assert.assertEquals(new Long(1), timestampCounts.get("4")); + Assert.assertEquals(new Long(1), timestampCounts.get("5")); + Assert.assertEquals(new Long(1), timestampCounts.get("6")); + Assert.assertEquals(new Long(1), timestampCounts.get("7")); + Assert.assertEquals(new Long(1), timestampCounts.get("8")); + Assert.assertEquals(new Long(1), timestampCounts.get("9")); + Assert.assertEquals(new Long(1), timestampCounts.get("10")); + Map<String, Long> latitudeCounts = facetCounts.get("latitude"); + Assert.assertEquals(2, latitudeCounts.size()); + List<String> latitudeKeys = new ArrayList<>(latitudeCounts.keySet()); + Collections.sort(latitudeKeys); + Assert.assertEquals(48.0001, Double.parseDouble(latitudeKeys.get(0)), 0.00001); + Assert.assertEquals(48.5839, Double.parseDouble(latitudeKeys.get(1)), 0.00001); + Assert.assertEquals(new Long(2), latitudeCounts.get(latitudeKeys.get(0))); + Assert.assertEquals(new Long(8), latitudeCounts.get(latitudeKeys.get(1))); + Map<String, Long> scoreFieldCounts = facetCounts.get("score"); + Assert.assertEquals(4, scoreFieldCounts.size()); + List<String> scoreFieldKeys = new ArrayList<>(scoreFieldCounts.keySet()); + Collections.sort(scoreFieldKeys); + Assert.assertEquals(10.0, Double.parseDouble(scoreFieldKeys.get(0)), 0.00001); + Assert.assertEquals(20.0, Double.parseDouble(scoreFieldKeys.get(1)), 0.00001); + Assert.assertEquals(50.0, Double.parseDouble(scoreFieldKeys.get(2)), 0.00001); + Assert.assertEquals(98.0, Double.parseDouble(scoreFieldKeys.get(3)), 0.00001); + Assert.assertEquals(new Long(4), scoreFieldCounts.get(scoreFieldKeys.get(0))); + Assert.assertEquals(new Long(2), scoreFieldCounts.get(scoreFieldKeys.get(1))); + Assert.assertEquals(new Long(3), scoreFieldCounts.get(scoreFieldKeys.get(2))); + Assert.assertEquals(new Long(1), scoreFieldCounts.get(scoreFieldKeys.get(3))); + Map<String, Long> isAlertCounts = facetCounts.get("is_alert"); + Assert.assertEquals(2, isAlertCounts.size()); + Assert.assertEquals(new Long(6), isAlertCounts.get("true")); + Assert.assertEquals(new Long(4), isAlertCounts.get("false")); + } + + @Test + public void bad_facet_query_throws_exception() throws Exception { + thrown.expect(InvalidSearchException.class); + thrown.expectMessage("Failed to execute search"); + SearchRequest request = JSONUtils.INSTANCE.load(badFacetQuery, SearchRequest.class); + dao.search(request); + } + + @Test + public void disabled_facet_query_returns_null_count() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(disabledFacetQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertNull(response.getFacetCounts()); + } + + @Test + public void exceeding_max_resulsts_throws_exception() throws Exception { + thrown.expect(InvalidSearchException.class); + thrown.expectMessage("Search result size must be less than 100"); + SearchRequest request = JSONUtils.INSTANCE.load(exceededMaxResultsQuery, SearchRequest.class); + dao.search(request); + } + + @Test + public void returns_column_data_for_multiple_indices() throws Exception { + Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Arrays.asList("bro", "snort")); + Assert.assertEquals(15, fieldTypes.size()); + Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type")); + Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr")); + Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port")); + Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field")); + Assert.assertEquals(FieldType.DATE, fieldTypes.get("timestamp")); + Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("latitude")); + Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score")); + Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert")); + Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field")); + Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field")); + //NOTE: This is because the field is in both bro and snort and they have different types. + Assert.assertEquals(FieldType.OTHER, fieldTypes.get("duplicate_name_field")); + Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("threat:triage:score")); + Assert.assertEquals(FieldType.OTHER, fieldTypes.get("alert")); + } + + @Test + public void returns_column_metadata_for_specified_indices() throws Exception { // getColumnMetadata with only bro { Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("bro")); Assert.assertEquals(13, fieldTypes.size()); - Assert.assertEquals(FieldType.STRING, fieldTypes.get("guid")); - Assert.assertEquals(FieldType.STRING, fieldTypes.get("source:type")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("duplicate_name_field")); + Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type")); Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr")); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port")); Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field")); @@ -724,8 +889,8 @@ public abstract class SearchIntegrationTest { Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score")); Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert")); Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point")); - Assert.assertEquals(FieldType.STRING, fieldTypes.get("bro_field")); - Assert.assertEquals(FieldType.STRING, fieldTypes.get("duplicate_name_field")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("duplicate_name_field")); Assert.assertEquals(FieldType.OTHER, fieldTypes.get("alert")); } // getColumnMetadata with only snort @@ -733,8 +898,9 @@ public abstract class SearchIntegrationTest { Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("snort")); Assert.assertEquals(14, fieldTypes.size()); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field")); - Assert.assertEquals(FieldType.STRING, fieldTypes.get("guid")); - Assert.assertEquals(FieldType.STRING, fieldTypes.get("source:type")); + Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field")); + Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type")); Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr")); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port")); Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field")); @@ -751,204 +917,208 @@ public abstract class SearchIntegrationTest { Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("someindex")); Assert.assertEquals(0, fieldTypes.size()); } - //Fields query - { - SearchRequest request = JSONUtils.INSTANCE.load(fieldsQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(10, response.getTotal()); - List<SearchResult> results = response.getResults(); - for(int i = 0;i < 5;++i) { - Map<String, Object> source = results.get(i).getSource(); - Assert.assertEquals(1, source.size()); - Assert.assertNotNull(source.get("ip_src_addr")); - } - for(int i = 5;i < 10;++i) { - Map<String, Object> source = results.get(i).getSource(); - Assert.assertEquals(1, source.size()); - Assert.assertNotNull(source.get("ip_src_addr")); - } - } - //Meta Alerts Fields query - { - SearchRequest request = JSONUtils.INSTANCE.load(metaAlertsFieldQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(2, response.getTotal()); - List<SearchResult> results = response.getResults(); - for (int i = 0;i < 2;++i) { - Map<String, Object> source = results.get(i).getSource(); - Assert.assertEquals(1, source.size()); - Assert.assertEquals(source.get("guid"), "meta_" + (i + 1)); - } - } - //No results fields query - { - SearchRequest request = JSONUtils.INSTANCE.load(noResultsFieldsQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(0, response.getTotal()); - } + } + + + @Test + public void group_by_ip_query() throws Exception { + GroupRequest request = JSONUtils.INSTANCE.load(groupByIpQuery, GroupRequest.class); + GroupResponse response = dao.group(request); + + // expect only 1 group for 'ip_src_addr' + Assert.assertEquals("ip_src_addr", response.getGroupedBy()); + + // there are 8 different 'ip_src_addr' values + List<GroupResult> groups = response.getGroupResults(); + Assert.assertEquals(8, groups.size()); + + // expect dotted-decimal notation in descending order + Assert.assertEquals("192.168.1.8", groups.get(0).getKey()); + Assert.assertEquals("192.168.1.7", groups.get(1).getKey()); + Assert.assertEquals("192.168.1.6", groups.get(2).getKey()); + Assert.assertEquals("192.168.1.5", groups.get(3).getKey()); + Assert.assertEquals("192.168.1.4", groups.get(4).getKey()); + Assert.assertEquals("192.168.1.3", groups.get(5).getKey()); + Assert.assertEquals("192.168.1.2", groups.get(6).getKey()); + Assert.assertEquals("192.168.1.1", groups.get(7).getKey()); + } + + @Test + public void no_results_returned_when_query_does_not_match() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(noResultsFieldsQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(0, response.getTotal()); + } + + @Test + public void group_by_returns_results_in_groups() throws Exception { // Group by test case, default order is count descending - { - GroupRequest request = JSONUtils.INSTANCE.load(groupByQuery, GroupRequest.class); - GroupResponse response = dao.group(request); - Assert.assertEquals("is_alert", response.getGroupedBy()); - List<GroupResult> isAlertGroups = response.getGroupResults(); - Assert.assertEquals(2, isAlertGroups.size()); - - // isAlert == true group - GroupResult trueGroup = isAlertGroups.get(0); - Assert.assertEquals("true", trueGroup.getKey()); - Assert.assertEquals(6, trueGroup.getTotal()); - Assert.assertEquals("latitude", trueGroup.getGroupedBy()); - Assert.assertEquals(198.0, trueGroup.getScore(), 0.00001); - List<GroupResult> trueLatitudeGroups = trueGroup.getGroupResults(); - Assert.assertEquals(2, trueLatitudeGroups.size()); - - // isAlert == true && latitude == 48.5839 group - GroupResult trueLatitudeGroup2 = trueLatitudeGroups.get(0); - Assert.assertEquals(48.5839, Double.parseDouble(trueLatitudeGroup2.getKey()), 0.00001); - Assert.assertEquals(5, trueLatitudeGroup2.getTotal()); - Assert.assertEquals(148.0, trueLatitudeGroup2.getScore(), 0.00001); - - // isAlert == true && latitude == 48.0001 group - GroupResult trueLatitudeGroup1 = trueLatitudeGroups.get(1); - Assert.assertEquals(48.0001, Double.parseDouble(trueLatitudeGroup1.getKey()), 0.00001); - Assert.assertEquals(1, trueLatitudeGroup1.getTotal()); - Assert.assertEquals(50.0, trueLatitudeGroup1.getScore(), 0.00001); - - // isAlert == false group - GroupResult falseGroup = isAlertGroups.get(1); - Assert.assertEquals("false", falseGroup.getKey()); - Assert.assertEquals("latitude", falseGroup.getGroupedBy()); - Assert.assertEquals(130.0, falseGroup.getScore(), 0.00001); - List<GroupResult> falseLatitudeGroups = falseGroup.getGroupResults(); - Assert.assertEquals(2, falseLatitudeGroups.size()); - - // isAlert == false && latitude == 48.5839 group - GroupResult falseLatitudeGroup2 = falseLatitudeGroups.get(0); - Assert.assertEquals(48.5839, Double.parseDouble(falseLatitudeGroup2.getKey()), 0.00001); - Assert.assertEquals(3, falseLatitudeGroup2.getTotal()); - Assert.assertEquals(80.0, falseLatitudeGroup2.getScore(), 0.00001); - - // isAlert == false && latitude == 48.0001 group - GroupResult falseLatitudeGroup1 = falseLatitudeGroups.get(1); - Assert.assertEquals(48.0001, Double.parseDouble(falseLatitudeGroup1.getKey()), 0.00001); - Assert.assertEquals(1, falseLatitudeGroup1.getTotal()); - Assert.assertEquals(50.0, falseLatitudeGroup1.getScore(), 0.00001); - } + GroupRequest request = JSONUtils.INSTANCE.load(groupByQuery, GroupRequest.class); + GroupResponse response = dao.group(request); + Assert.assertEquals("is_alert", response.getGroupedBy()); + List<GroupResult> isAlertGroups = response.getGroupResults(); + Assert.assertEquals(2, isAlertGroups.size()); + + // isAlert == true group + GroupResult trueGroup = isAlertGroups.get(0); + Assert.assertEquals("true", trueGroup.getKey()); + Assert.assertEquals(6, trueGroup.getTotal()); + Assert.assertEquals("latitude", trueGroup.getGroupedBy()); + Assert.assertEquals(198.0, trueGroup.getScore(), 0.00001); + List<GroupResult> trueLatitudeGroups = trueGroup.getGroupResults(); + Assert.assertEquals(2, trueLatitudeGroups.size()); + + + // isAlert == true && latitude == 48.5839 group + GroupResult trueLatitudeGroup2 = trueLatitudeGroups.get(0); + Assert.assertEquals(48.5839, Double.parseDouble(trueLatitudeGroup2.getKey()), 0.00001); + Assert.assertEquals(5, trueLatitudeGroup2.getTotal()); + Assert.assertEquals(148.0, trueLatitudeGroup2.getScore(), 0.00001); + + // isAlert == true && latitude == 48.0001 group + GroupResult trueLatitudeGroup1 = trueLatitudeGroups.get(1); + Assert.assertEquals(48.0001, Double.parseDouble(trueLatitudeGroup1.getKey()), 0.00001); + Assert.assertEquals(1, trueLatitudeGroup1.getTotal()); + Assert.assertEquals(50.0, trueLatitudeGroup1.getScore(), 0.00001); + + // isAlert == false group + GroupResult falseGroup = isAlertGroups.get(1); + Assert.assertEquals("false", falseGroup.getKey()); + Assert.assertEquals("latitude", falseGroup.getGroupedBy()); + Assert.assertEquals(130.0, falseGroup.getScore(), 0.00001); + List<GroupResult> falseLatitudeGroups = falseGroup.getGroupResults(); + Assert.assertEquals(2, falseLatitudeGroups.size()); + + // isAlert == false && latitude == 48.5839 group + GroupResult falseLatitudeGroup2 = falseLatitudeGroups.get(0); + Assert.assertEquals(48.5839, Double.parseDouble(falseLatitudeGroup2.getKey()), 0.00001); + Assert.assertEquals(3, falseLatitudeGroup2.getTotal()); + Assert.assertEquals(80.0, falseLatitudeGroup2.getScore(), 0.00001); + + // isAlert == false && latitude == 48.0001 group + GroupResult falseLatitudeGroup1 = falseLatitudeGroups.get(1); + Assert.assertEquals(48.0001, Double.parseDouble(falseLatitudeGroup1.getKey()), 0.00001); + Assert.assertEquals(1, falseLatitudeGroup1.getTotal()); + Assert.assertEquals(50.0, falseLatitudeGroup1.getScore(), 0.00001); + } + + @Test + public void group_by_returns_results_in_sorted_groups() throws Exception { // Group by with sorting test case where is_alert is sorted by count ascending and ip_src_addr is sorted by term descending - { - GroupRequest request = JSONUtils.INSTANCE.load(sortedGroupByQuery, GroupRequest.class); - GroupResponse response = dao.group(request); - Assert.assertEquals("is_alert", response.getGroupedBy()); - List<GroupResult> isAlertGroups = response.getGroupResults(); - Assert.assertEquals(2, isAlertGroups.size()); - - // isAlert == false group - GroupResult falseGroup = isAlertGroups.get(0); - Assert.assertEquals(4, falseGroup.getTotal()); - Assert.assertEquals("ip_src_addr", falseGroup.getGroupedBy()); - List<GroupResult> falseIpSrcAddrGroups = falseGroup.getGroupResults(); - Assert.assertEquals(4, falseIpSrcAddrGroups.size()); - - // isAlert == false && ip_src_addr == 192.168.1.8 group - GroupResult falseIpSrcAddrGroup1 = falseIpSrcAddrGroups.get(0); - Assert.assertEquals("192.168.1.8", falseIpSrcAddrGroup1.getKey()); - Assert.assertEquals(1, falseIpSrcAddrGroup1.getTotal()); - Assert.assertNull(falseIpSrcAddrGroup1.getGroupedBy()); - Assert.assertNull(falseIpSrcAddrGroup1.getGroupResults()); - - // isAlert == false && ip_src_addr == 192.168.1.7 group - GroupResult falseIpSrcAddrGroup2 = falseIpSrcAddrGroups.get(1); - Assert.assertEquals("192.168.1.7", falseIpSrcAddrGroup2.getKey()); - Assert.assertEquals(1, falseIpSrcAddrGroup2.getTotal()); - Assert.assertNull(falseIpSrcAddrGroup2.getGroupedBy()); - Assert.assertNull(falseIpSrcAddrGroup2.getGroupResults()); - - // isAlert == false && ip_src_addr == 192.168.1.6 group - GroupResult falseIpSrcAddrGroup3 = falseIpSrcAddrGroups.get(2); - Assert.assertEquals("192.168.1.6", falseIpSrcAddrGroup3.getKey()); - Assert.assertEquals(1, falseIpSrcAddrGroup3.getTotal()); - Assert.assertNull(falseIpSrcAddrGroup3.getGroupedBy()); - Assert.assertNull(falseIpSrcAddrGroup3.getGroupResults()); - - // isAlert == false && ip_src_addr == 192.168.1.2 group - GroupResult falseIpSrcAddrGroup4 = falseIpSrcAddrGroups.get(3); - Assert.assertEquals("192.168.1.2", falseIpSrcAddrGroup4.getKey()); - Assert.assertEquals(1, falseIpSrcAddrGroup4.getTotal()); - Assert.assertNull(falseIpSrcAddrGroup4.getGroupedBy()); - Assert.assertNull(falseIpSrcAddrGroup4.getGroupResults()); - - // isAlert == false group - GroupResult trueGroup = isAlertGroups.get(1); - Assert.assertEquals(6, trueGroup.getTotal()); - Assert.assertEquals("ip_src_addr", trueGroup.getGroupedBy()); - List<GroupResult> trueIpSrcAddrGroups = trueGroup.getGroupResults(); - Assert.assertEquals(4, trueIpSrcAddrGroups.size()); - - // isAlert == false && ip_src_addr == 192.168.1.5 group - GroupResult trueIpSrcAddrGroup1 = trueIpSrcAddrGroups.get(0); - Assert.assertEquals("192.168.1.5", trueIpSrcAddrGroup1.getKey()); - Assert.assertEquals(1, trueIpSrcAddrGroup1.getTotal()); - Assert.assertNull(trueIpSrcAddrGroup1.getGroupedBy()); - Assert.assertNull(trueIpSrcAddrGroup1.getGroupResults()); - - // isAlert == false && ip_src_addr == 192.168.1.4 group - GroupResult trueIpSrcAddrGroup2 = trueIpSrcAddrGroups.get(1); - Assert.assertEquals("192.168.1.4", trueIpSrcAddrGroup2.getKey()); - Assert.assertEquals(1, trueIpSrcAddrGroup2.getTotal()); - Assert.assertNull(trueIpSrcAddrGroup2.getGroupedBy()); - Assert.assertNull(trueIpSrcAddrGroup2.getGroupResults()); - - // isAlert == false && ip_src_addr == 192.168.1.3 group - GroupResult trueIpSrcAddrGroup3 = trueIpSrcAddrGroups.get(2); - Assert.assertEquals("192.168.1.3", trueIpSrcAddrGroup3.getKey()); - Assert.assertEquals(1, trueIpSrcAddrGroup3.getTotal()); - Assert.assertNull(trueIpSrcAddrGroup3.getGroupedBy()); - Assert.assertNull(trueIpSrcAddrGroup3.getGroupResults()); - - // isAlert == false && ip_src_addr == 192.168.1.1 group - GroupResult trueIpSrcAddrGroup4 = trueIpSrcAddrGroups.get(3); - Assert.assertEquals("192.168.1.1", trueIpSrcAddrGroup4.getKey()); - Assert.assertEquals(3, trueIpSrcAddrGroup4.getTotal()); - Assert.assertNull(trueIpSrcAddrGroup4.getGroupedBy()); - Assert.assertNull(trueIpSrcAddrGroup4.getGroupResults()); + GroupRequest request = JSONUtils.INSTANCE.load(sortedGroupByQuery, GroupRequest.class); + GroupResponse response = dao.group(request); + Assert.assertEquals("is_alert", response.getGroupedBy()); + List<GroupResult> isAlertGroups = response.getGroupResults(); + Assert.assertEquals(2, isAlertGroups.size()); + + // isAlert == false group + GroupResult falseGroup = isAlertGroups.get(0); + Assert.assertEquals(4, falseGroup.getTotal()); + Assert.assertEquals("ip_src_addr", falseGroup.getGroupedBy()); + List<GroupResult> falseIpSrcAddrGroups = falseGroup.getGroupResults(); + Assert.assertEquals(4, falseIpSrcAddrGroups.size()); + + // isAlert == false && ip_src_addr == 192.168.1.8 group + GroupResult falseIpSrcAddrGroup1 = falseIpSrcAddrGroups.get(0); + Assert.assertEquals("192.168.1.8", falseIpSrcAddrGroup1.getKey()); + Assert.assertEquals(1, falseIpSrcAddrGroup1.getTotal()); + Assert.assertNull(falseIpSrcAddrGroup1.getGroupedBy()); + Assert.assertNull(falseIpSrcAddrGroup1.getGroupResults()); + + // isAlert == false && ip_src_addr == 192.168.1.7 group + GroupResult falseIpSrcAddrGroup2 = falseIpSrcAddrGroups.get(1); + Assert.assertEquals("192.168.1.7", falseIpSrcAddrGroup2.getKey()); + Assert.assertEquals(1, falseIpSrcAddrGroup2.getTotal()); + Assert.assertNull(falseIpSrcAddrGroup2.getGroupedBy()); + Assert.assertNull(falseIpSrcAddrGroup2.getGroupResults()); + + // isAlert == false && ip_src_addr == 192.168.1.6 group + GroupResult falseIpSrcAddrGroup3 = falseIpSrcAddrGroups.get(2); + Assert.assertEquals("192.168.1.6", falseIpSrcAddrGroup3.getKey()); + Assert.assertEquals(1, falseIpSrcAddrGroup3.getTotal()); + Assert.assertNull(falseIpSrcAddrGroup3.getGroupedBy()); + Assert.assertNull(falseIpSrcAddrGroup3.getGroupResults()); + + // isAlert == false && ip_src_addr == 192.168.1.2 group + GroupResult falseIpSrcAddrGroup4 = falseIpSrcAddrGroups.get(3); + Assert.assertEquals("192.168.1.2", falseIpSrcAddrGroup4.getKey()); + Assert.assertEquals(1, falseIpSrcAddrGroup4.getTotal()); + Assert.assertNull(falseIpSrcAddrGroup4.getGroupedBy()); + Assert.assertNull(falseIpSrcAddrGroup4.getGroupResults()); + + // isAlert == false group + GroupResult trueGroup = isAlertGroups.get(1); + Assert.assertEquals(6, trueGroup.getTotal()); + Assert.assertEquals("ip_src_addr", trueGroup.getGroupedBy()); + List<GroupResult> trueIpSrcAddrGroups = trueGroup.getGroupResults(); + Assert.assertEquals(4, trueIpSrcAddrGroups.size()); + + // isAlert == false && ip_src_addr == 192.168.1.5 group + GroupResult trueIpSrcAddrGroup1 = trueIpSrcAddrGroups.get(0); + Assert.assertEquals("192.168.1.5", trueIpSrcAddrGroup1.getKey()); + Assert.assertEquals(1, trueIpSrcAddrGroup1.getTotal()); + Assert.assertNull(trueIpSrcAddrGroup1.getGroupedBy()); + Assert.assertNull(trueIpSrcAddrGroup1.getGroupResults()); + + // isAlert == false && ip_src_addr == 192.168.1.4 group + GroupResult trueIpSrcAddrGroup2 = trueIpSrcAddrGroups.get(1); + Assert.assertEquals("192.168.1.4", trueIpSrcAddrGroup2.getKey()); + Assert.assertEquals(1, trueIpSrcAddrGroup2.getTotal()); + Assert.assertNull(trueIpSrcAddrGroup2.getGroupedBy()); + Assert.assertNull(trueIpSrcAddrGroup2.getGroupResults()); + + // isAlert == false && ip_src_addr == 192.168.1.3 group + GroupResult trueIpSrcAddrGroup3 = trueIpSrcAddrGroups.get(2); + Assert.assertEquals("192.168.1.3", trueIpSrcAddrGroup3.getKey()); + Assert.assertEquals(1, trueIpSrcAddrGroup3.getTotal()); + Assert.assertNull(trueIpSrcAddrGroup3.getGroupedBy()); + Assert.assertNull(trueIpSrcAddrGroup3.getGroupResults()); + + // isAlert == false && ip_src_addr == 192.168.1.1 group + GroupResult trueIpSrcAddrGroup4 = trueIpSrcAddrGroups.get(3); + Assert.assertEquals("192.168.1.1", trueIpSrcAddrGroup4.getKey()); + Assert.assertEquals(3, trueIpSrcAddrGroup4.getTotal()); + Assert.assertNull(trueIpSrcAddrGroup4.getGroupedBy()); + Assert.assertNull(trueIpSrcAddrGroup4.getGroupResults()); + } + + @Test + public void throws_exception_on_aggregation_queries_on_non_string_non_numeric_fields() + throws Exception { + thrown.expect(InvalidSearchException.class); + thrown.expectMessage("Failed to execute search"); + GroupRequest request = JSONUtils.INSTANCE.load(badGroupQuery, GroupRequest.class); + dao.group(request); + } + + @Test + public void queries_fields() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(fieldsQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + List<SearchResult> results = response.getResults(); + for (int i = 0; i < 5; ++i) { + Map<String, Object> source = results.get(i).getSource(); + Assert.assertEquals(1, source.size()); + Assert.assertNotNull(source.get("ip_src_addr")); } - //Bad group query - { - GroupRequest request = JSONUtils.INSTANCE.load(badGroupQuery, GroupRequest.class); - try { - dao.group(request); - Assert.fail("Exception expected, but did not come."); - } - catch(InvalidSearchException ise) { - // success - } + for (int i = 5; i < 10; ++i) { + Map<String, Object> source = results.get(i).getSource(); + Assert.assertEquals(1, source.size()); + Assert.assertNotNull(source.get("ip_src_addr")); } - //Group by IP query - { - { - GroupRequest request = JSONUtils.INSTANCE.load(groupByIpQuery, GroupRequest.class); - GroupResponse response = dao.group(request); - - // expect only 1 group for 'ip_src_addr' - Assert.assertEquals("ip_src_addr", response.getGroupedBy()); - - // there are 8 different 'ip_src_addr' values - List<GroupResult> groups = response.getGroupResults(); - Assert.assertEquals(8, groups.size()); - - // expect dotted-decimal notation in descending order - Assert.assertEquals("192.168.1.8", groups.get(0).getKey()); - Assert.assertEquals("192.168.1.7", groups.get(1).getKey()); - Assert.assertEquals("192.168.1.6", groups.get(2).getKey()); - Assert.assertEquals("192.168.1.5", groups.get(3).getKey()); - Assert.assertEquals("192.168.1.4", groups.get(4).getKey()); - Assert.assertEquals("192.168.1.3", groups.get(5).getKey()); - Assert.assertEquals("192.168.1.2", groups.get(6).getKey()); - Assert.assertEquals("192.168.1.1", groups.get(7).getKey()); - } - + } + @Test + public void searches_metaalerts_fields() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(metaAlertsFieldQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(2, response.getTotal()); + List<SearchResult> results = response.getResults(); + for (int i = 0; i < 2; ++i) { + Map<String, Object> source = results.get(i).getSource(); + Assert.assertEquals(1, source.size()); + Assert.assertEquals(source.get("guid"), "meta_" + (i + 1)); } } @@ -960,4 +1130,4 @@ public abstract class SearchIntegrationTest { protected abstract IndexDao createDao() throws Exception; protected abstract InMemoryComponent startIndex() throws Exception; protected abstract void loadTestData() throws Exception; -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java index c0f9919..ac6f90a 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java @@ -56,8 +56,8 @@ public abstract class IndexingIntegrationTest extends BaseIntegrationTest { protected String sampleParsedPath = TestConstants.SAMPLE_DATA_PARSED_PATH + "TestExampleParsed"; protected String fluxPath = "../metron-indexing/src/main/flux/indexing/remote.yaml"; protected String testSensorType = "test"; - - + protected final int NUM_RETRIES = 100; + protected final long TOTAL_TIME_MS = 150000L; public static List<Map<String, Object>> readDocsFromDisk(String hdfsDirStr) throws IOException { List<Map<String, Object>> ret = new ArrayList<>(); File hdfsDir = new File(hdfsDirStr); @@ -180,8 +180,8 @@ public abstract class IndexingIntegrationTest extends BaseIntegrationTest { .withComponent("storm", fluxComponent) .withComponent("search", getSearchComponent(topologyProperties)) .withMillisecondsBetweenAttempts(1500) - .withNumRetries(100) - .withMaxTimeMS(150000) + .withNumRetries(NUM_RETRIES) + .withMaxTimeMS(TOTAL_TIME_MS) .withCustomShutdownOrder(new String[] {"search","storm","config","kafka","zk"}) .build(); @@ -198,8 +198,6 @@ public abstract class IndexingIntegrationTest extends BaseIntegrationTest { // on the field name converter assertInputDocsMatchOutputs(inputDocs, docs, getFieldNameConverter()); assertInputDocsMatchOutputs(inputDocs, readDocsFromDisk(hdfsDir), x -> x); - } catch(Throwable e) { - e.printStackTrace(); } finally { if(runner != null) { http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-platform/metron-integration-test/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-integration-test/pom.xml b/metron-platform/metron-integration-test/pom.xml index 72a785c..cc72849 100644 --- a/metron-platform/metron-integration-test/pom.xml +++ b/metron-platform/metron-integration-test/pom.xml @@ -53,6 +53,14 @@ <artifactId>log4j-slf4j-impl</artifactId> <groupId>org.apache.logging.log4j</groupId> </exclusion> + <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-api</artifactId> + </exclusion> </exclusions> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/TestUtils.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/TestUtils.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/TestUtils.java index 9577a43..0c37a35 100644 --- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/TestUtils.java +++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/TestUtils.java @@ -21,10 +21,12 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; @@ -32,6 +34,7 @@ import java.util.List; public class TestUtils { public static long MAX_ASSERT_WAIT_MS = 30000L; + public interface Assertion { void apply() throws Exception; } @@ -88,6 +91,30 @@ public class TestUtils { } /** + * Reads file contents into a String. Uses UTF-8 as default charset. + * + * @param in Input file + * @return contents of input file + * @throws IOException + */ + public static String read(File in) throws IOException { + return read(in, StandardCharsets.UTF_8); + } + + /** + * Reads file contents into a String + * + * @param in Input file + * @param charset charset to use for reading + * @return contents of input file + * @throws IOException + */ + public static String read(File in, Charset charset) throws IOException { + byte[] bytes = Files.readAllBytes(Paths.get(in.getPath())); + return new String(bytes, charset); + } + + /** * Cleans up after test run via runtime shutdown hooks */ public static File createTempDir(String prefix) throws IOException { http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-platform/metron-parsers/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml index c27ad62..18377d3 100644 --- a/metron-platform/metron-parsers/pom.xml +++ b/metron-platform/metron-parsers/pom.xml @@ -130,6 +130,14 @@ <scope>provided</scope> <exclusions> <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + </exclusion> + <exclusion> <artifactId>servlet-api</artifactId> <groupId>javax.servlet</groupId> </exclusion> @@ -144,6 +152,18 @@ </exclusions> </dependency> <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-api</artifactId> + <version>${global_log4j_core_version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + <version>${global_log4j_core_version}</version> + <scope>test</scope> + </dependency> + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <version>${global_junit_version}</version> http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-platform/metron-pcap-backend/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap-backend/pom.xml b/metron-platform/metron-pcap-backend/pom.xml index bcd7e7c..6db7d29 100644 --- a/metron-platform/metron-pcap-backend/pom.xml +++ b/metron-platform/metron-pcap-backend/pom.xml @@ -174,6 +174,14 @@ <scope>provided</scope> <exclusions> <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + </exclusion> + <exclusion> <artifactId>servlet-api</artifactId> <groupId>javax.servlet</groupId> </exclusion> @@ -188,6 +196,18 @@ </exclusions> </dependency> <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-api</artifactId> + <version>${global_log4j_core_version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + <version>${global_log4j_core_version}</version> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.apache.metron</groupId> <artifactId>metron-pcap</artifactId> <version>${project.parent.version}</version> http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-platform/metron-pcap/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap/pom.xml b/metron-platform/metron-pcap/pom.xml index 3a83174..574e99b 100644 --- a/metron-platform/metron-pcap/pom.xml +++ b/metron-platform/metron-pcap/pom.xml @@ -92,6 +92,14 @@ <scope>provided</scope> <exclusions> <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + </exclusion> + <exclusion> <artifactId>servlet-api</artifactId> <groupId>javax.servlet</groupId> </exclusion> http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-platform/metron-solr/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/pom.xml b/metron-platform/metron-solr/pom.xml index 6da67b8..a2eee71 100644 --- a/metron-platform/metron-solr/pom.xml +++ b/metron-platform/metron-solr/pom.xml @@ -66,12 +66,32 @@ </exclusions> </dependency> <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-api</artifactId> + <version>${global_log4j_core_version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + <version>${global_log4j_core_version}</version> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.apache.storm</groupId> <artifactId>storm-core</artifactId> <version>${global_storm_version}</version> <scope>provided</scope> <exclusions> <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-api</artifactId> + </exclusion> + <exclusion> <artifactId>servlet-api</artifactId> <groupId>javax.servlet</groupId> </exclusion>
