This is an automated email from the ASF dual-hosted git repository.
pvillard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 23d177be3e NIFI-14491 Support other types in
CreateBoxFileMetadataInstance
23d177be3e is described below
commit 23d177be3eb6efd98488c1b070726310a874cbbb
Author: Alaksiej Ščarbaty <[email protected]>
AuthorDate: Tue Apr 22 17:54:10 2025 +0200
NIFI-14491 Support other types in CreateBoxFileMetadataInstance
Signed-off-by: Pierre Villard <[email protected]>
This closes #9891.
---
.../box/CreateBoxFileMetadataInstance.java | 56 +++++++++++++++++++---
.../box/UpdateBoxFileMetadataInstance.java | 18 ++++++-
.../apache/nifi/processors/box/utils/BoxDate.java | 43 +++++++++++++++++
.../box/CreateBoxFileMetadataInstanceTest.java | 28 ++++++++++-
.../box/UpdateBoxFileMetadataInstanceTest.java | 5 +-
.../nifi/processors/box/utils/BoxDateTest.java | 40 ++++++++++++++++
6 files changed, 179 insertions(+), 11 deletions(-)
diff --git
a/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/CreateBoxFileMetadataInstance.java
b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/CreateBoxFileMetadataInstance.java
index 55667cade0..c686d34dc2 100644
---
a/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/CreateBoxFileMetadataInstance.java
+++
b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/CreateBoxFileMetadataInstance.java
@@ -37,14 +37,20 @@ import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.processors.box.utils.BoxDate;
import org.apache.nifi.serialization.RecordReader;
import org.apache.nifi.serialization.RecordReaderFactory;
import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordField;
+import org.apache.nifi.serialization.record.RecordFieldType;
import java.io.InputStream;
+import java.time.LocalDate;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import static java.lang.String.valueOf;
@@ -223,20 +229,58 @@ public class CreateBoxFileMetadataInstance extends
AbstractProcessor {
return;
}
- List<String> fieldNames = record.getSchema().getFieldNames();
+ final List<RecordField> fields = record.getSchema().getFields();
- if (fieldNames.isEmpty()) {
+ if (fields.isEmpty()) {
errors.add("Record has no fields");
return;
}
- for (String fieldName : fieldNames) {
- Object valueObj = record.getValue(fieldName);
- String value = valueObj != null ? valueObj.toString() : null;
- metadata.add("/" + fieldName, value);
+ for (final RecordField field : fields) {
+ addValueToMetadata(metadata, record, field);
}
}
+ private void addValueToMetadata(final Metadata metadata, final Record
record, final RecordField field) {
+ if (record.getValue(field) == null) {
+ return;
+ }
+
+ final RecordFieldType fieldType = field.getDataType().getFieldType();
+ final String fieldName = field.getFieldName();
+ final String path = "/" + fieldName;
+
+ if (isNumber(fieldType)) {
+ metadata.add(path, record.getAsDouble(fieldName));
+ } else if (isDate(fieldType)) {
+ final LocalDate date = record.getAsLocalDate(fieldName, null);
+ metadata.add(path, BoxDate.of(date).format());
+ } else if (isArray(fieldType)) {
+ final List<String> values =
Arrays.stream(record.getAsArray(fieldName))
+ .filter(Objects::nonNull)
+ .map(Object::toString)
+ .toList();
+
+ metadata.add(path, values);
+ } else {
+ metadata.add(path, record.getAsString(fieldName));
+ }
+ }
+
+ private boolean isNumber(final RecordFieldType fieldType) {
+ final boolean isInteger = RecordFieldType.BIGINT.equals(fieldType) ||
RecordFieldType.BIGINT.isWiderThan(fieldType);
+ final boolean isFloat = RecordFieldType.DECIMAL.equals(fieldType) ||
RecordFieldType.DECIMAL.isWiderThan(fieldType);
+ return isInteger || isFloat;
+ }
+
+ private boolean isDate(final RecordFieldType fieldType) {
+ return RecordFieldType.DATE.equals(fieldType);
+ }
+
+ private boolean isArray(final RecordFieldType fieldType) {
+ return RecordFieldType.ARRAY.equals(fieldType);
+ }
+
/**
* Returns a BoxFile object for the given file ID.
*
diff --git
a/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/UpdateBoxFileMetadataInstance.java
b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/UpdateBoxFileMetadataInstance.java
index 0f4fda7449..c8ea194ba6 100644
---
a/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/UpdateBoxFileMetadataInstance.java
+++
b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/UpdateBoxFileMetadataInstance.java
@@ -37,11 +37,15 @@ import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.processors.box.utils.BoxDate;
import org.apache.nifi.serialization.RecordReader;
import org.apache.nifi.serialization.RecordReaderFactory;
import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordField;
+import org.apache.nifi.serialization.record.RecordFieldType;
import java.io.InputStream;
+import java.time.LocalDate;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -216,8 +220,16 @@ public class UpdateBoxFileMetadataInstance extends
AbstractProcessor {
final Record record = recordReader.nextRecord();
if (record != null) {
- for (String fieldName : record.getSchema().getFieldNames()) {
- desiredState.put(fieldName, record.getValue(fieldName));
+ final List<RecordField> fields =
record.getSchema().getFields();
+ for (final RecordField field : fields) {
+ final String fieldName = field.getFieldName();
+ final RecordFieldType type =
field.getDataType().getFieldType();
+
+ final Object value = RecordFieldType.DATE.equals(type)
+ ? record.getAsLocalDate(fieldName, null) //
Ensuring dates are read as LocalDate.
+ : record.getValue(field);
+
+ desiredState.put(fieldName, value);
}
}
}
@@ -269,6 +281,7 @@ public class UpdateBoxFileMetadataInstance extends
AbstractProcessor {
switch (value) {
case Number n -> metadata.replace(propertyPath,
n.doubleValue());
case List<?> l -> metadata.replace(propertyPath,
convertListToStringList(l, propertyPath));
+ case LocalDate d -> metadata.replace(propertyPath,
BoxDate.of(d).format());
default -> metadata.replace(propertyPath, value.toString());
}
} else {
@@ -276,6 +289,7 @@ public class UpdateBoxFileMetadataInstance extends
AbstractProcessor {
switch (value) {
case Number n -> metadata.add(propertyPath, n.doubleValue());
case List<?> l -> metadata.add(propertyPath,
convertListToStringList(l, propertyPath));
+ case LocalDate d -> metadata.add(propertyPath,
BoxDate.of(d).format());
default -> metadata.add(propertyPath, value.toString());
}
}
diff --git
a/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/utils/BoxDate.java
b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/utils/BoxDate.java
new file mode 100644
index 0000000000..2cb9bcf988
--- /dev/null
+++
b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/utils/BoxDate.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.box.utils;
+
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+
+/**
+ * A wrapper class for formatting {@link LocalDate} to a string that is
accepted by Box Metadata API.
+ */
+public final class BoxDate {
+
+ // The time part is always set to 0.
https://developer.box.com/guides/metadata/fields/date/
+ private static final DateTimeFormatter DATE_FORMATTER =
DateTimeFormatter.ofPattern("yyyy-MM-dd'T00:00:00.000Z'");
+
+ private final LocalDate date;
+
+ private BoxDate(final LocalDate date) {
+ this.date = date;
+ }
+
+ public static BoxDate of(final LocalDate date) {
+ return new BoxDate(date);
+ }
+
+ public String format() {
+ return DATE_FORMATTER.format(date);
+ }
+}
diff --git
a/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/CreateBoxFileMetadataInstanceTest.java
b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/CreateBoxFileMetadataInstanceTest.java
index eb7d71a86c..959518a25c 100644
---
a/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/CreateBoxFileMetadataInstanceTest.java
+++
b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/CreateBoxFileMetadataInstanceTest.java
@@ -31,6 +31,13 @@ import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
+import java.text.ParseException;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.util.Date;
+import java.util.List;
+
+import static java.time.ZoneOffset.UTC;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doThrow;
@@ -75,14 +82,20 @@ public class CreateBoxFileMetadataInstanceTest extends
AbstractBoxFileTest {
}
@Test
- void testSuccessfulMetadataCreation() {
+ void testSuccessfulMetadataCreation() throws ParseException {
final String inputJson = """
{
"audience": "internal",
"documentType": "Q1 plans",
"competitiveDocument": "no",
"status": "active",
- "author": "Jones"
+ "author": "Jones",
+ "int": 1,
+ "double": 1.234,
+ "almostTenToThePowerOfThirty":
1000000000000000000000000000123,
+ "array": [ "one", "two", "three" ],
+ "intArray": [ 1, 2, 3 ],
+ "date": "2025-01-01"
}""";
testRunner.enqueue(inputJson);
@@ -97,6 +110,12 @@ public class CreateBoxFileMetadataInstanceTest extends
AbstractBoxFileTest {
assertEquals("no",
capturedMetadata.getValue("/competitiveDocument").asString());
assertEquals("active",
capturedMetadata.getValue("/status").asString());
assertEquals("Jones", capturedMetadata.getValue("/author").asString());
+ assertEquals(1, capturedMetadata.getValue("/int").asInt());
+ assertEquals(1.234, capturedMetadata.getDouble("/double"));
+ assertEquals(1e30,
capturedMetadata.getDouble("/almostTenToThePowerOfThirty")); // Precision loss
is accepted.
+ assertEquals(List.of("one", "two", "three"),
capturedMetadata.getMultiSelect("/array"));
+ assertEquals(List.of("1", "2", "3"),
capturedMetadata.getMultiSelect("/intArray"));
+ assertEquals(createLegacyDate(2025, 1, 1),
capturedMetadata.getDate("/date"));
testRunner.assertAllFlowFilesTransferred(CreateBoxFileMetadataInstance.REL_SUCCESS,
1);
final MockFlowFile flowFile =
testRunner.getFlowFilesForRelationship(CreateBoxFileMetadataInstance.REL_SUCCESS).getFirst();
@@ -157,4 +176,9 @@ public class CreateBoxFileMetadataInstanceTest extends
AbstractBoxFileTest {
flowFile.assertAttributeEquals(BoxFileAttributes.ERROR_MESSAGE, "API
Error [404]");
}
+ private static Date createLegacyDate(int year, int month, int day) {
+ final LocalDate date = LocalDate.of(year, month, day);
+ final Instant instant = date.atStartOfDay(UTC).toInstant();
+ return Date.from(instant);
+ }
}
diff --git
a/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/UpdateBoxFileMetadataInstanceTest.java
b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/UpdateBoxFileMetadataInstanceTest.java
index 0f7437083b..fd6f349c68 100644
---
a/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/UpdateBoxFileMetadataInstanceTest.java
+++
b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/UpdateBoxFileMetadataInstanceTest.java
@@ -91,6 +91,7 @@ public class UpdateBoxFileMetadataInstanceTest extends
AbstractBoxFileTest {
private void configureJsonRecordReader(TestRunner runner) throws
InitializationException {
final JsonTreeReader readerService = new JsonTreeReader();
runner.addControllerService("json-reader", readerService);
+ runner.setProperty(readerService, "Date Format", "yyyy-MM-dd");
runner.enableControllerService(readerService);
}
@@ -290,7 +291,8 @@ public class UpdateBoxFileMetadataInstanceTest extends
AbstractBoxFileTest {
"doubleField": 42.5,
"booleanField": true,
"listField": ["item1", "item2", "item3"],
- "emptyListField": []
+ "emptyListField": [],
+ "date": "2025-01-01"
}""";
testRunner.enqueue(inputJson);
@@ -301,6 +303,7 @@ public class UpdateBoxFileMetadataInstanceTest extends
AbstractBoxFileTest {
verify(mockMetadata).add("/numberField", 42.0); // Numbers are stored
as doubles
verify(mockMetadata).add("/doubleField", 42.5);
verify(mockMetadata).add("/booleanField", "true"); // Booleans are
stored as strings
+ verify(mockMetadata).add("/date", "2025-01-01T00:00:00.000Z"); //
Dates have a specific format.
// We need to use doAnswer/when to capture and verify list fields
being added, but this is simpler
verify(mockBoxFile).updateMetadata(any(Metadata.class));
diff --git
a/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/utils/BoxDateTest.java
b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/utils/BoxDateTest.java
new file mode 100644
index 0000000000..ce6ebcccaa
--- /dev/null
+++
b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/utils/BoxDateTest.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.box.utils;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+import java.time.LocalDate;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+class BoxDateTest {
+
+ @ParameterizedTest
+ @CsvSource({
+ "2025-01-01, 2025-01-01T00:00:00.000Z",
+ "2025-02-25, 2025-02-25T00:00:00.000Z",
+ "2025-11-10, 2025-11-10T00:00:00.000Z",
+ })
+ void format(final LocalDate date, final String expected) {
+ final BoxDate boxDate = BoxDate.of(date);
+ final String formatted = boxDate.format();
+
+ assertEquals(expected, formatted);
+ }
+}
