This is an automated email from the ASF dual-hosted git repository. chriss pushed a commit to branch support/nifi-1.x in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push: new 13cb524cb5 NIFI-11778 fix nifi-elasticsearch-restapi-processor tests for NiFi 1.x 13cb524cb5 is described below commit 13cb524cb51e1c5e569240a2bd958b58193ad6c5 Author: Chris Sampson <chris.sampso...@gmail.com> AuthorDate: Tue Aug 29 07:02:05 2023 +0100 NIFI-11778 fix nifi-elasticsearch-restapi-processor tests for NiFi 1.x --- .../AbstractByQueryElasticsearchTest.java | 7 +++---- .../AbstractJsonQueryElasticsearchTest.java | 10 ++++----- ...bstractPaginatedJsonQueryElasticsearchTest.java | 7 +++---- .../elasticsearch/JsonQueryElasticsearchTest.java | 6 ++++-- .../nifi/processors/elasticsearch/JsonUtils.java | 11 ++++++++++ .../elasticsearch/PutElasticsearchRecordTest.java | 24 ++++++++++------------ .../TestElasticsearchClientService.java | 5 ++--- 7 files changed, 39 insertions(+), 31 deletions(-) diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/AbstractByQueryElasticsearchTest.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/AbstractByQueryElasticsearchTest.java index deb0998039..f6c059b88e 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/AbstractByQueryElasticsearchTest.java +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/AbstractByQueryElasticsearchTest.java @@ -23,7 +23,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.nio.file.Files; import java.nio.file.Paths; import java.util.Collections; import java.util.List; @@ -53,7 +52,7 @@ public abstract class AbstractByQueryElasticsearchTest { @BeforeAll public static void setUpBeforeClass() throws Exception { - matchAllQuery = Files.readString(Paths.get("src/test/resources/common/matchAllQuery.json")); + matchAllQuery = JsonUtils.readString(Paths.get("src/test/resources/common/matchAllQuery.json")); } @BeforeEach @@ -114,7 +113,7 @@ public abstract class AbstractByQueryElasticsearchTest { @Test public void testWithQuery() throws Exception { - final String query = Files.readString(Paths.get(TEST_DIR,"matchUsingExpressionLanguageQuery.json")); + final String query = JsonUtils.readString(Paths.get(TEST_DIR,"matchUsingExpressionLanguageQuery.json")); runner.setProperty(AbstractByQueryElasticsearch.QUERY, query); runner.setProperty(AbstractByQueryElasticsearch.INDEX, INDEX); runner.setProperty(AbstractByQueryElasticsearch.TYPE, TYPE); @@ -127,7 +126,7 @@ public abstract class AbstractByQueryElasticsearchTest { runner.clearTransferState(); - final String query2 = Files.readString(Paths.get(TEST_DIR, "matchQuery.json")); + final String query2 = JsonUtils.readString(Paths.get(TEST_DIR, "matchQuery.json")); runner.setProperty(AbstractByQueryElasticsearch.QUERY, query2); runner.setIncomingConnection(false); runner.assertValid(); diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/AbstractJsonQueryElasticsearchTest.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/AbstractJsonQueryElasticsearchTest.java index 70f6d21785..df8a8c4a1a 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/AbstractJsonQueryElasticsearchTest.java +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/AbstractJsonQueryElasticsearchTest.java @@ -18,6 +18,7 @@ package org.apache.nifi.processors.elasticsearch; import org.apache.nifi.components.state.Scope; import org.apache.nifi.processors.elasticsearch.api.AggregationResultsFormat; +import org.apache.nifi.processors.elasticsearch.api.JsonQueryParameters; import org.apache.nifi.processors.elasticsearch.api.ResultOutputStrategy; import org.apache.nifi.processors.elasticsearch.api.SearchResultsFormat; import org.apache.nifi.provenance.ProvenanceEventType; @@ -29,7 +30,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Paths; import java.util.Arrays; import java.util.List; @@ -43,7 +43,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -public abstract class AbstractJsonQueryElasticsearchTest<P extends AbstractJsonQueryElasticsearch> { +public abstract class AbstractJsonQueryElasticsearchTest<P extends AbstractJsonQueryElasticsearch<? extends JsonQueryParameters>> { private static final String TEST_DIR = "src/test/resources/AbstractJsonQueryElasticsearchTest"; private static final String TEST_COMMON_DIR = "src/test/resources/common"; private static final String INDEX_NAME = "messages"; @@ -58,8 +58,8 @@ public abstract class AbstractJsonQueryElasticsearchTest<P extends AbstractJsonQ @BeforeAll public static void setUpBeforeClass() throws Exception { - matchAllQuery = Files.readString(Paths.get(TEST_COMMON_DIR, "matchAllQuery.json")); - matchAllAggregationWithDefaultTermsQuery = Files.readString(Paths.get(TEST_DIR,"matchAllAggregationWithDefaultTermsQuery.json")); + matchAllQuery = JsonUtils.readString(Paths.get(TEST_COMMON_DIR, "matchAllQuery.json")); + matchAllAggregationWithDefaultTermsQuery = JsonUtils.readString(Paths.get(TEST_DIR,"matchAllAggregationWithDefaultTermsQuery.json")); } @Test @@ -296,7 +296,7 @@ public abstract class AbstractJsonQueryElasticsearchTest<P extends AbstractJsonQ @Test public void testAggregationsUsingExpressionLanguage() throws Exception { final TestRunner runner = createRunner(true); - String query = Files.readString(Paths.get(TEST_DIR, "matchAllAggregationWithDefaultTermsInExpressionLanguageQuery.json")); + String query = JsonUtils.readString(Paths.get(TEST_DIR, "matchAllAggregationWithDefaultTermsInExpressionLanguageQuery.json")); runner.setVariable("fieldValue", "msg"); runner.setVariable("es.index", INDEX_NAME); runner.setVariable("es.type", "msg"); diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/AbstractPaginatedJsonQueryElasticsearchTest.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/AbstractPaginatedJsonQueryElasticsearchTest.java index e3d25f8d79..3ec4d43ad4 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/AbstractPaginatedJsonQueryElasticsearchTest.java +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/AbstractPaginatedJsonQueryElasticsearchTest.java @@ -26,7 +26,6 @@ import org.apache.nifi.util.TestRunner; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import java.nio.file.Files; import java.nio.file.Paths; import java.util.Map; import java.util.stream.Collectors; @@ -46,9 +45,9 @@ public abstract class AbstractPaginatedJsonQueryElasticsearchTest extends Abstra @BeforeAll public static void setUpBeforeClass() throws Exception { AbstractJsonQueryElasticsearchTest.setUpBeforeClass(); - matchAllWithSortByMessage = Files.readString(Paths.get(TEST_DIR, "matchAllWithSortByMessageQuery.json")); - matchAllWithSortByMsgWithoutSize = Files.readString(Paths.get(TEST_DIR,"matchAllWithSortByMsgQueryWithoutSize.json")); - matchAllWithSortByMsgWithSizeQuery = Files.readString(Paths.get(TEST_DIR, "matchAllWithSortByMsgQueryWithSize.json")); + matchAllWithSortByMessage = JsonUtils.readString(Paths.get(TEST_DIR, "matchAllWithSortByMessageQuery.json")); + matchAllWithSortByMsgWithoutSize = JsonUtils.readString(Paths.get(TEST_DIR,"matchAllWithSortByMsgQueryWithoutSize.json")); + matchAllWithSortByMsgWithSizeQuery = JsonUtils.readString(Paths.get(TEST_DIR, "matchAllWithSortByMsgQueryWithSize.json")); } public abstract boolean isInput(); diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/JsonQueryElasticsearchTest.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/JsonQueryElasticsearchTest.java index 5325854e61..6dfbd23237 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/JsonQueryElasticsearchTest.java +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/JsonQueryElasticsearchTest.java @@ -16,9 +16,11 @@ */ package org.apache.nifi.processors.elasticsearch; -public class JsonQueryElasticsearchTest extends AbstractJsonQueryElasticsearchTest<AbstractJsonQueryElasticsearch> { +import org.apache.nifi.processors.elasticsearch.api.JsonQueryParameters; + +public class JsonQueryElasticsearchTest extends AbstractJsonQueryElasticsearchTest<AbstractJsonQueryElasticsearch<JsonQueryParameters>> { @Override - public AbstractJsonQueryElasticsearch getProcessor() { + public AbstractJsonQueryElasticsearch<JsonQueryParameters> getProcessor() { return new JsonQueryElasticsearch(); } diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/JsonUtils.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/JsonUtils.java index 6d2b3794a3..a1d097a976 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/JsonUtils.java +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/JsonUtils.java @@ -19,9 +19,13 @@ package org.apache.nifi.processors.elasticsearch; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import java.util.stream.Stream; public class JsonUtils { private static final ObjectMapper MAPPER = new ObjectMapper(); @@ -43,6 +47,7 @@ public class JsonUtils { } } + @SuppressWarnings("unchecked") static Map<String, Object> readMap(String json) { try { return MAPPER.readValue(json, Map.class); @@ -51,12 +56,18 @@ public class JsonUtils { } } + static String readString(final Path path) throws IOException { + try (Stream<String> lines = Files.lines(path)) { + return lines.collect(Collectors.joining("\n")); + } + } static List<String> readListOfMapsAsIndividualJson(String json) { return readListOfMaps(json).stream() .map(JsonUtils::prettyPrint) .collect(Collectors.toList()); } + @SuppressWarnings("unchecked") static List<Map<String, Object>> readListOfMaps(String json) { try { return MAPPER.readValue(json, List.class); diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchRecordTest.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchRecordTest.java index 32cb9b6df4..954bf314e7 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchRecordTest.java +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchRecordTest.java @@ -39,7 +39,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.sql.Date; @@ -89,10 +88,9 @@ public class PutElasticsearchRecordTest extends AbstractPutElasticsearchTest<Put public Class<? extends AbstractPutElasticsearch> getTestProcessor() { return PutElasticsearchRecord.class; } - @BeforeAll public static void setUpBeforeClass() throws Exception { - flowFileContentMaps = Files.readString(Paths.get(TEST_DIR, "flowFileContentMaps.json")); + flowFileContentMaps = JsonUtils.readString(Paths.get(TEST_DIR, "flowFileContentMaps.json")); simpleSchema = getRecordSchema(Paths.get(TEST_DIR, "simpleSchema.json")); recordPathTestSchema = getRecordSchema(Paths.get(TEST_DIR, "recordPathTestSchema.json")); dateTimeFormattingTestSchema = getRecordSchema(Paths.get(TEST_DIR, "dateTimeFormattingTestSchema.json")); @@ -240,9 +238,9 @@ public class PutElasticsearchRecordTest extends AbstractPutElasticsearchTest<Put @Test public void testRecordPathFeatures() throws Exception { Map<String, Object> script = - JsonUtils.readMap(Files.readString(Paths.get(TEST_DIR, "script.json"))); + JsonUtils.readMap(JsonUtils.readString(Paths.get(TEST_DIR, "script.json"))); Map<String, Object> dynamicTemplates = - JsonUtils.readMap(Files.readString(Paths.get(TEST_COMMON_DIR, "dynamicTemplates.json"))); + JsonUtils.readMap(JsonUtils.readString(Paths.get(TEST_COMMON_DIR, "dynamicTemplates.json"))); clientService.setEvalConsumer((List<IndexOperationRequest> items) -> { long a = items.stream().filter(item -> "bulk_a".equals(item.getIndex())).count(); long b = items.stream().filter(item -> "bulk_b".equals(item.getIndex())).count(); @@ -297,7 +295,7 @@ public class PutElasticsearchRecordTest extends AbstractPutElasticsearchTest<Put runner.setProperty(PutElasticsearchRecord.SCRIPT_RECORD_PATH, "/script"); runner.setProperty(PutElasticsearchRecord.SCRIPTED_UPSERT_RECORD_PATH, "/scripted_upsert"); runner.setProperty(PutElasticsearchRecord.DYNAMIC_TEMPLATES_RECORD_PATH, "/dynamic_templates"); - String flowFileContents = Files.readString(Paths.get(TEST_DIR, "1_flowFileContents.json")); + String flowFileContents = JsonUtils.readString(Paths.get(TEST_DIR, "1_flowFileContents.json")); flowFileContents = flowFileContents.replaceFirst("\\d{13}", String.valueOf(Timestamp.valueOf(LOCAL_DATE_TIME).toInstant().toEpochMilli())); runner.enqueue(flowFileContents, Collections.singletonMap(SCHEMA_NAME_ATTRIBUTE, RECORD_PATH_TEST_SCHEMA)); @@ -313,7 +311,7 @@ public class PutElasticsearchRecordTest extends AbstractPutElasticsearchTest<Put @Test public void testTimestampDateFormatAndScriptRecordPath() throws Exception { Map<String, Object> script = - JsonUtils.readMap(Files.readString(Paths.get(TEST_DIR, "script.json"))); + JsonUtils.readMap(JsonUtils.readString(Paths.get(TEST_DIR, "script.json"))); clientService.setEvalConsumer((List<IndexOperationRequest> items) -> { long testTypeCount = items.stream().filter(item -> "test_type".equals(item.getType())).count(); long messageTypeCount = items.stream().filter(item -> "message".equals(item.getType())).count(); @@ -356,7 +354,7 @@ public class PutElasticsearchRecordTest extends AbstractPutElasticsearchTest<Put Map<String, String> attributes = new LinkedHashMap<>(); attributes.put(SCHEMA_NAME_ATTRIBUTE, RECORD_PATH_TEST_SCHEMA); attributes.put("operation", "index"); - String flowFileContents = Files.readString(Paths.get(TEST_DIR, "2_flowFileContents.json")); + String flowFileContents = JsonUtils.readString(Paths.get(TEST_DIR, "2_flowFileContents.json")); flowFileContents = flowFileContents.replaceFirst("\\d{13}", String.valueOf(Date.valueOf(LOCAL_DATE).getTime())); runner.enqueue(flowFileContents, attributes); runner.run(); @@ -391,7 +389,7 @@ public class PutElasticsearchRecordTest extends AbstractPutElasticsearchTest<Put runner.removeProperty(PutElasticsearchRecord.TYPE); runner.setProperty(PutElasticsearchRecord.ID_RECORD_PATH, "/id"); runner.setProperty(PutElasticsearchRecord.TYPE_RECORD_PATH, "/type"); - String flowFileContents = Files.readString(Paths.get(TEST_DIR, "3_flowFileContents.json")); + String flowFileContents = JsonUtils.readString(Paths.get(TEST_DIR, "3_flowFileContents.json")); flowFileContents = flowFileContents.replaceFirst("\\d{8}", String.valueOf(Time.valueOf(LOCAL_TIME).getTime())); runner.enqueue(flowFileContents, Collections.singletonMap(SCHEMA_NAME_ATTRIBUTE, RECORD_PATH_TEST_SCHEMA)); runner.run(); @@ -767,7 +765,7 @@ public class PutElasticsearchRecordTest extends AbstractPutElasticsearchTest<Put } private static RecordSchema getRecordSchema(Path schema) throws IOException { - return AvroTypeUtil.createSchema(new Schema.Parser().parse(Files.readString(schema))); + return AvroTypeUtil.createSchema(new Schema.Parser().parse(JsonUtils.readString(schema))); } private void testErrorRelationship(final int errorCount, final int successCount, boolean recordWriter) throws Exception { @@ -777,7 +775,7 @@ public class PutElasticsearchRecordTest extends AbstractPutElasticsearchTest<Put runner.setProperty(writer, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_NAME_PROPERTY); runner.setProperty(writer, SchemaAccessUtils.SCHEMA_REGISTRY, "registry"); runner.enableControllerService(writer); - clientService.setResponse(IndexOperationResponse.fromJsonResponse(Files.readString(Paths.get(TEST_COMMON_DIR, "sampleErrorResponse.json")))); + clientService.setResponse(IndexOperationResponse.fromJsonResponse(JsonUtils.readString(Paths.get(TEST_COMMON_DIR, "sampleErrorResponse.json")))); registry.addSchema(schemaName, errorTestSchema); if(recordWriter) { @@ -795,7 +793,7 @@ public class PutElasticsearchRecordTest extends AbstractPutElasticsearchTest<Put assertEquals(1, runner.getProvenanceEvents().stream().filter(e -> ProvenanceEventType.SEND.equals(e.getEventType()) - && "1 Elasticsearch _bulk operation batch(es) [%s error(s), %s success(es)]".formatted(errorCount, successCount).equals(e.getDetails())).count()); + && String.format("1 Elasticsearch _bulk operation batch(es) [%s error(s), %s success(es)]", errorCount, successCount).equals(e.getDetails())).count()); } private void testInvalidELRecordPaths(String idRecordPath, String atTimestampRecordPath, Path path, Map<String, String> attributes) throws IOException { @@ -848,7 +846,7 @@ public class PutElasticsearchRecordTest extends AbstractPutElasticsearchTest<Put } private String getDateTimeFormattingJson() throws Exception { - String json = Files.readString(Paths.get(TEST_DIR, "10_flowFileContents.json")); + String json = JsonUtils.readString(Paths.get(TEST_DIR, "10_flowFileContents.json")); List<Map<String, Object>> parsedJson = JsonUtils.readListOfMaps(json); parsedJson.forEach(msg -> { msg.computeIfPresent("ts", (key, val) -> Timestamp.valueOf(LOCAL_DATE_TIME).toInstant().toEpochMilli()); diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/TestElasticsearchClientService.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/TestElasticsearchClientService.java index 4ce3808834..1a09c147d9 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/TestElasticsearchClientService.java +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/TestElasticsearchClientService.java @@ -29,7 +29,6 @@ import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.processors.elasticsearch.mock.MockElasticsearchException; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collections; @@ -43,8 +42,8 @@ public class TestElasticsearchClientService extends AbstractControllerService im static { try { - AGGS_RESULT = Files.readString(Paths.get("src/test/resources/TestElasticsearchClientService/aggsResult.json")); - HITS_RESULT = Files.readString(Paths.get("src/test/resources/TestElasticsearchClientService/hitsResult.json")); + AGGS_RESULT = JsonUtils.readString(Paths.get("src/test/resources/TestElasticsearchClientService/aggsResult.json")); + HITS_RESULT = JsonUtils.readString(Paths.get("src/test/resources/TestElasticsearchClientService/hitsResult.json")); } catch (IOException e) { throw new RuntimeException(e); }