Github user nickwallen commented on a diff in the pull request: https://github.com/apache/metron/pull/1247#discussion_r236643962 --- Diff: metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java --- @@ -97,45 +118,63 @@ protected static InMemoryComponent startIndex() throws Exception { return es; } - protected static void loadTestData() throws ParseException, IOException { + protected static void loadTestData() throws Exception { ElasticSearchComponent es = (ElasticSearchComponent) indexComponent; + // define the bro index template + String broIndex = "bro_index_2017.01.01.01"; JSONObject broTemplate = JSONUtils.INSTANCE.load(new File(broTemplatePath), JSONObject.class); addTestFieldMappings(broTemplate, "bro_doc"); - es.getClient().admin().indices().prepareCreate("bro_index_2017.01.01.01") - .addMapping("bro_doc", JSONUtils.INSTANCE.toJSON(broTemplate.get("mappings"), false)).get(); + es.getClient().admin().indices().prepareCreate(broIndex) + .addMapping("bro_doc", JSONUtils.INSTANCE.toJSON(broTemplate.get("mappings"), false)).get(); + + // define the snort index template + String snortIndex = "snort_index_2017.01.01.02"; JSONObject snortTemplate = JSONUtils.INSTANCE.load(new File(snortTemplatePath), JSONObject.class); addTestFieldMappings(snortTemplate, "snort_doc"); - es.getClient().admin().indices().prepareCreate("snort_index_2017.01.01.02") - .addMapping("snort_doc", JSONUtils.INSTANCE.toJSON(snortTemplate.get("mappings"), false)).get(); - - BulkRequestBuilder bulkRequest = es.getClient().prepareBulk() - .setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); - JSONArray broArray = (JSONArray) new JSONParser().parse(broData); - for (Object o : broArray) { - JSONObject jsonObject = (JSONObject) o; - IndexRequestBuilder indexRequestBuilder = es.getClient() - .prepareIndex("bro_index_2017.01.01.01", "bro_doc"); - indexRequestBuilder = indexRequestBuilder.setId((String) jsonObject.get("guid")); - indexRequestBuilder = indexRequestBuilder.setSource(jsonObject.toJSONString()); - indexRequestBuilder = indexRequestBuilder - .setTimestamp(jsonObject.get("timestamp").toString()); - bulkRequest.add(indexRequestBuilder); + es.getClient().admin().indices().prepareCreate(snortIndex) + .addMapping("snort_doc", JSONUtils.INSTANCE.toJSON(snortTemplate.get("mappings"), false)).get(); + + // setup the classes required to write the test data + AccessConfig accessConfig = createAccessConfig(); + ElasticsearchClient client = ElasticsearchUtils.getClient(createGlobalConfig()); + ElasticsearchRetrieveLatestDao retrieveLatestDao = new ElasticsearchRetrieveLatestDao(client); + ElasticsearchColumnMetadataDao columnMetadataDao = new ElasticsearchColumnMetadataDao(client); + ElasticsearchRequestSubmitter requestSubmitter = new ElasticsearchRequestSubmitter(client); + ElasticsearchUpdateDao updateDao = new ElasticsearchUpdateDao(client, accessConfig, retrieveLatestDao); + ElasticsearchSearchDao searchDao = new ElasticsearchSearchDao(client, accessConfig, columnMetadataDao, requestSubmitter); + + // write the test documents for Bro + List<String> broDocuments = new ArrayList<>(); + for (Object broObject: (JSONArray) new JSONParser().parse(broData)) { + broDocuments.add(((JSONObject) broObject).toJSONString()); } - JSONArray snortArray = (JSONArray) new JSONParser().parse(snortData); - for (Object o : snortArray) { - JSONObject jsonObject = (JSONObject) o; - IndexRequestBuilder indexRequestBuilder = es.getClient() - .prepareIndex("snort_index_2017.01.01.02", "snort_doc"); - indexRequestBuilder = indexRequestBuilder.setId((String) jsonObject.get("guid")); - indexRequestBuilder = indexRequestBuilder.setSource(jsonObject.toJSONString()); - indexRequestBuilder = indexRequestBuilder - .setTimestamp(jsonObject.get("timestamp").toString()); - bulkRequest.add(indexRequestBuilder); + es.add(updateDao, broIndex, "bro", broDocuments); + + // write the test documents for Snort + List<String> snortDocuments = new ArrayList<>(); + for (Object snortObject: (JSONArray) new JSONParser().parse(snortData)) { + snortDocuments.add(((JSONObject) snortObject).toJSONString()); } - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); - if (bulkResponse.hasFailures()) { - throw new RuntimeException("Failed to index test data"); + es.add(updateDao, snortIndex, "snort", snortDocuments); + + // wait until the test documents are visible + assertEventually(() -> Assert.assertEquals(10, findAll(searchDao).getTotal())); --- End diff -- > Are you able to get this to fail consistently? I ran it a number of times and couldn't get it to fail. Can you describe what you did? You just removed line 190 and the tests continue to work reliably?
---