Github user mmiklavc commented on a diff in the pull request:
https://github.com/apache/metron/pull/1247#discussion_r236524985
--- Diff:
metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
---
@@ -97,45 +118,63 @@ protected static InMemoryComponent startIndex() throws
Exception {
return es;
}
- protected static void loadTestData() throws ParseException, IOException {
+ protected static void loadTestData() throws Exception {
ElasticSearchComponent es = (ElasticSearchComponent) indexComponent;
+ // define the bro index template
+ String broIndex = "bro_index_2017.01.01.01";
JSONObject broTemplate = JSONUtils.INSTANCE.load(new
File(broTemplatePath), JSONObject.class);
addTestFieldMappings(broTemplate, "bro_doc");
-
es.getClient().admin().indices().prepareCreate("bro_index_2017.01.01.01")
- .addMapping("bro_doc",
JSONUtils.INSTANCE.toJSON(broTemplate.get("mappings"), false)).get();
+ es.getClient().admin().indices().prepareCreate(broIndex)
+ .addMapping("bro_doc",
JSONUtils.INSTANCE.toJSON(broTemplate.get("mappings"), false)).get();
+
+ // define the snort index template
+ String snortIndex = "snort_index_2017.01.01.02";
JSONObject snortTemplate = JSONUtils.INSTANCE.load(new
File(snortTemplatePath), JSONObject.class);
addTestFieldMappings(snortTemplate, "snort_doc");
-
es.getClient().admin().indices().prepareCreate("snort_index_2017.01.01.02")
- .addMapping("snort_doc",
JSONUtils.INSTANCE.toJSON(snortTemplate.get("mappings"), false)).get();
-
- BulkRequestBuilder bulkRequest = es.getClient().prepareBulk()
- .setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
- JSONArray broArray = (JSONArray) new JSONParser().parse(broData);
- for (Object o : broArray) {
- JSONObject jsonObject = (JSONObject) o;
- IndexRequestBuilder indexRequestBuilder = es.getClient()
- .prepareIndex("bro_index_2017.01.01.01", "bro_doc");
- indexRequestBuilder = indexRequestBuilder.setId((String)
jsonObject.get("guid"));
- indexRequestBuilder =
indexRequestBuilder.setSource(jsonObject.toJSONString());
- indexRequestBuilder = indexRequestBuilder
- .setTimestamp(jsonObject.get("timestamp").toString());
- bulkRequest.add(indexRequestBuilder);
+ es.getClient().admin().indices().prepareCreate(snortIndex)
+ .addMapping("snort_doc",
JSONUtils.INSTANCE.toJSON(snortTemplate.get("mappings"), false)).get();
+
+ // setup the classes required to write the test data
+ AccessConfig accessConfig = createAccessConfig();
+ ElasticsearchClient client =
ElasticsearchUtils.getClient(createGlobalConfig());
+ ElasticsearchRetrieveLatestDao retrieveLatestDao = new
ElasticsearchRetrieveLatestDao(client);
+ ElasticsearchColumnMetadataDao columnMetadataDao = new
ElasticsearchColumnMetadataDao(client);
+ ElasticsearchRequestSubmitter requestSubmitter = new
ElasticsearchRequestSubmitter(client);
+ ElasticsearchUpdateDao updateDao = new ElasticsearchUpdateDao(client,
accessConfig, retrieveLatestDao);
+ ElasticsearchSearchDao searchDao = new ElasticsearchSearchDao(client,
accessConfig, columnMetadataDao, requestSubmitter);
+
+ // write the test documents for Bro
+ List<String> broDocuments = new ArrayList<>();
+ for (Object broObject: (JSONArray) new JSONParser().parse(broData)) {
+ broDocuments.add(((JSONObject) broObject).toJSONString());
}
- JSONArray snortArray = (JSONArray) new JSONParser().parse(snortData);
- for (Object o : snortArray) {
- JSONObject jsonObject = (JSONObject) o;
- IndexRequestBuilder indexRequestBuilder = es.getClient()
- .prepareIndex("snort_index_2017.01.01.02", "snort_doc");
- indexRequestBuilder = indexRequestBuilder.setId((String)
jsonObject.get("guid"));
- indexRequestBuilder =
indexRequestBuilder.setSource(jsonObject.toJSONString());
- indexRequestBuilder = indexRequestBuilder
- .setTimestamp(jsonObject.get("timestamp").toString());
- bulkRequest.add(indexRequestBuilder);
+ es.add(updateDao, broIndex, "bro", broDocuments);
+
+ // write the test documents for Snort
+ List<String> snortDocuments = new ArrayList<>();
+ for (Object snortObject: (JSONArray) new
JSONParser().parse(snortData)) {
+ snortDocuments.add(((JSONObject) snortObject).toJSONString());
}
- BulkResponse bulkResponse = bulkRequest.execute().actionGet();
- if (bulkResponse.hasFailures()) {
- throw new RuntimeException("Failed to index test data");
+ es.add(updateDao, snortIndex, "snort", snortDocuments);
+
+ // wait until the test documents are visible
+ assertEventually(() -> Assert.assertEquals(10,
findAll(searchDao).getTotal()));
--- End diff --
Are you able to get this to fail consistently? I ran it a number of times
and couldn't get it to fail. I'm wondering if there are any other interesting
entries in the logs when this occurs. These tests from ES might be of help -
https://github.com/elastic/elasticsearch/pull/17986/files#diff-1c9f982dbd2f9ddb2853d135884621b5R112
---
