pvillard31 commented on code in PR #11355:
URL: https://github.com/apache/nifi/pull/11355#discussion_r3451092253
##########
nifi-extension-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchJsonTest.java:
##########
@@ -1110,4 +1110,170 @@ void testIndexFieldNullValueFallsBackConsistently() {
runner.run();
assertEquals("test_index", ops.getFirst().getIndex(), "NDJSON null ->
fallback (not literal \"null\")");
}
+
+ //
-------------------------------------------------------------------------
+ // Nested field paths (e.g. @metadata/index)
+ //
-------------------------------------------------------------------------
+
+ @Test
+ void testNestedIndexFieldNdjsonPrunesEmptyParent() {
+ runner.setProperty(PutElasticsearchJson.INPUT_FORMAT,
InputFormat.NDJSON.getValue());
+ runner.setProperty(PutElasticsearchJson.INDEX_FIELD,
"@metadata/index");
+ runner.setProperty(PutElasticsearchJson.RETAIN_INDEX_FIELD, "false");
+ runner.assertValid();
+
+ final List<IndexOperationRequest> ops = captureOperations();
+
runner.enqueue("{\"@metadata\":{\"index\":\"docs-2026\"},\"msg\":\"hello\"}\n");
+ runner.run();
+
+ assertEquals("docs-2026", ops.getFirst().getIndex());
+ final String content = docContent(ops.getFirst());
+ assertFalse(content.contains("@metadata"), "now-empty parent should be
pruned");
+ assertTrue(content.contains("hello"));
+ }
+
+ @Test
+ void testNestedIndexFieldJsonArrayPrunesEmptyParent() {
+ runner.setProperty(PutElasticsearchJson.INPUT_FORMAT,
InputFormat.JSON_ARRAY.getValue());
+ runner.setProperty(PutElasticsearchJson.INDEX_FIELD,
"@metadata/index");
+ runner.setProperty(PutElasticsearchJson.RETAIN_INDEX_FIELD, "false");
+ runner.assertValid();
+
+ final List<IndexOperationRequest> ops = captureOperations();
+
runner.enqueue("[{\"@metadata\":{\"index\":\"docs-a\"},\"msg\":\"x\"}]");
+ runner.run();
+
+ assertEquals("docs-a", ops.getFirst().getIndex());
+ assertFalse(docContent(ops.getFirst()).contains("@metadata"));
+ }
+
+ @Test
+ void testNestedIndexFieldSingleJsonPrunesEmptyParent() {
+ runner.setProperty(PutElasticsearchJson.INDEX_FIELD,
"@metadata/index");
+ runner.setProperty(PutElasticsearchJson.RETAIN_INDEX_FIELD, "false");
+ runner.assertValid();
+
+ final List<IndexOperationRequest> ops = captureOperations();
+
runner.enqueue("{\"@metadata\":{\"index\":\"docs-single\"},\"msg\":\"x\"}");
+ runner.run();
+
+ assertEquals("docs-single", ops.getFirst().getIndex());
+ assertFalse(ops.getFirst().getFields().containsKey("@metadata"));
+ }
+
+ @Test
+ void testNestedIndexFieldKeepsParentWithRemainingSiblings() {
+ // @metadata still holds another field after the index leaf is
removed, so it must NOT be pruned
+ runner.setProperty(PutElasticsearchJson.INPUT_FORMAT,
InputFormat.NDJSON.getValue());
+ runner.setProperty(PutElasticsearchJson.INDEX_FIELD,
"@metadata/index");
+ runner.setProperty(PutElasticsearchJson.RETAIN_INDEX_FIELD, "false");
+ runner.assertValid();
+
+ final List<IndexOperationRequest> ops = captureOperations();
+
runner.enqueue("{\"@metadata\":{\"index\":\"docs-2026\",\"keep\":\"yes\"},\"msg\":\"hello\"}\n");
+ runner.run();
+
+ assertEquals("docs-2026", ops.getFirst().getIndex());
+ final String content = docContent(ops.getFirst());
+ assertTrue(content.contains("@metadata"), "parent kept because it
still has siblings");
+ assertTrue(content.contains("keep"));
+ assertFalse(content.contains("\"index\""));
+ }
+
+ @Test
+ void testNestedIndexFieldRetainedByDefault() {
+ runner.setProperty(PutElasticsearchJson.INPUT_FORMAT,
InputFormat.NDJSON.getValue());
+ runner.setProperty(PutElasticsearchJson.INDEX_FIELD,
"@metadata/index");
+ runner.assertValid();
+
+ final List<IndexOperationRequest> ops = captureOperations();
+
runner.enqueue("{\"@metadata\":{\"index\":\"docs-2026\"},\"msg\":\"hello\"}\n");
+ runner.run();
+
+ assertEquals("docs-2026", ops.getFirst().getIndex());
+ assertTrue(docContent(ops.getFirst()).contains("@metadata"), "nested
field retained by default");
+ }
+
+ @Test
+ void testNestedIdentifierFieldNdjsonPrunesEmptyParent() {
Review Comment:
Should we add a nested identifier test for an Update, Delete, or Upsert
operation, so the resolveId nested path is covered as well?
##########
nifi-extension-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchJsonTest.java:
##########
@@ -1110,4 +1110,170 @@ void testIndexFieldNullValueFallsBackConsistently() {
runner.run();
assertEquals("test_index", ops.getFirst().getIndex(), "NDJSON null ->
fallback (not literal \"null\")");
}
+
+ //
-------------------------------------------------------------------------
+ // Nested field paths (e.g. @metadata/index)
+ //
-------------------------------------------------------------------------
+
+ @Test
+ void testNestedIndexFieldNdjsonPrunesEmptyParent() {
+ runner.setProperty(PutElasticsearchJson.INPUT_FORMAT,
InputFormat.NDJSON.getValue());
+ runner.setProperty(PutElasticsearchJson.INDEX_FIELD,
"@metadata/index");
+ runner.setProperty(PutElasticsearchJson.RETAIN_INDEX_FIELD, "false");
+ runner.assertValid();
+
+ final List<IndexOperationRequest> ops = captureOperations();
+
runner.enqueue("{\"@metadata\":{\"index\":\"docs-2026\"},\"msg\":\"hello\"}\n");
+ runner.run();
+
+ assertEquals("docs-2026", ops.getFirst().getIndex());
+ final String content = docContent(ops.getFirst());
+ assertFalse(content.contains("@metadata"), "now-empty parent should be
pruned");
+ assertTrue(content.contains("hello"));
+ }
+
+ @Test
+ void testNestedIndexFieldJsonArrayPrunesEmptyParent() {
+ runner.setProperty(PutElasticsearchJson.INPUT_FORMAT,
InputFormat.JSON_ARRAY.getValue());
+ runner.setProperty(PutElasticsearchJson.INDEX_FIELD,
"@metadata/index");
+ runner.setProperty(PutElasticsearchJson.RETAIN_INDEX_FIELD, "false");
+ runner.assertValid();
+
+ final List<IndexOperationRequest> ops = captureOperations();
+
runner.enqueue("[{\"@metadata\":{\"index\":\"docs-a\"},\"msg\":\"x\"}]");
+ runner.run();
+
+ assertEquals("docs-a", ops.getFirst().getIndex());
+ assertFalse(docContent(ops.getFirst()).contains("@metadata"));
+ }
+
+ @Test
+ void testNestedIndexFieldSingleJsonPrunesEmptyParent() {
+ runner.setProperty(PutElasticsearchJson.INDEX_FIELD,
"@metadata/index");
+ runner.setProperty(PutElasticsearchJson.RETAIN_INDEX_FIELD, "false");
+ runner.assertValid();
+
+ final List<IndexOperationRequest> ops = captureOperations();
+
runner.enqueue("{\"@metadata\":{\"index\":\"docs-single\"},\"msg\":\"x\"}");
+ runner.run();
+
+ assertEquals("docs-single", ops.getFirst().getIndex());
+ assertFalse(ops.getFirst().getFields().containsKey("@metadata"));
+ }
+
+ @Test
+ void testNestedIndexFieldKeepsParentWithRemainingSiblings() {
Review Comment:
Could we add a test with a 3 level path (for example a/b/c) to lock in the
recursive pruning of multiple empty ancestor objects?
##########
nifi-extension-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchJson.java:
##########
@@ -1143,30 +1156,168 @@ private String resolveId(final Map<String, Object>
contentMap, final String idAt
/**
* Extracts the index name from a pre-parsed {@link JsonNode}.
* Used for JSON Array Index/Create operations where the node is already
available.
+ * The field may be a {@code /}-delimited path into nested objects.
* Falls back to {@code fallbackIndex} when the field is absent or blank.
*/
private String extractIndex(final JsonNode node, final String indexField,
final String fallbackIndex) {
if (StringUtils.isBlank(indexField)) {
return fallbackIndex;
}
- final String value = fieldNodeToString(node.get(indexField));
+ final String value = fieldNodeToString(nodeAtPath(node, indexField));
return StringUtils.isNotBlank(value) ? value : fallbackIndex;
}
/**
* Resolves the index name from an already-parsed content Map.
* Used for Update/Delete/Upsert operations and suppression-enabled
Index/Create paths
- * where the Map is already available. Falls back to {@code fallbackIndex}
when the
- * field is absent or blank.
+ * where the Map is already available. The field may be a {@code
/}-delimited path into
+ * nested objects. Falls back to {@code fallbackIndex} when the field is
absent or blank.
*/
private String resolveIndex(final Map<String, Object> contentMap, final
String indexField, final String fallbackIndex) {
if (StringUtils.isBlank(indexField)) {
return fallbackIndex;
}
- final String value = fieldValueToString(contentMap.get(indexField));
+ final String value = fieldValueToString(valueAtPath(contentMap,
indexField));
return StringUtils.isNotBlank(value) ? value : fallbackIndex;
}
+ /**
+ * Whether {@code field} is a {@code /}-delimited nested path rather than
a flat field name.
+ */
+ private static boolean isNestedPath(final String field) {
Review Comment:
A field key that literally contains a slash is now always read as a nested
path. Should we note this limitation in the property descriptions so users with
such field names are not surprised?
##########
nifi-extension-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchJson.java:
##########
@@ -241,6 +241,7 @@ public class PutElasticsearchJson extends
AbstractPutElasticsearch {
.name("Identifier Field")
.description("""
The name of the field within each document to use as the
Elasticsearch document ID. \
+ A nested field can be referenced with a "/"-delimited path
(e.g. "@metadata/id"). \
Review Comment:
Could you rebase this branch on the latest main? The validation gate is
failing on "branch not up to date", and because of that the full build and test
matrix was skipped, so we do not have a green CI signal yet.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
