This is an automated email from the ASF dual-hosted git repository. mattyb149 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/master by this push: new e394f66 NIFI-6753: Fixed bug where all values being provided to the CSV Writer were String objects, which resulted in the CSV Writer improperly quoting numeric values when the schema indicates that the value is a number. Now, we will only convert the value to a String if the value is not a Number and/or the schema does not indicate a numeric ty type e394f66 is described below commit e394f6683aaadc20f59ab467ab781e50c86ba958 Author: Mark Payne <marka...@hotmail.com> AuthorDate: Wed Oct 9 09:22:18 2019 -0400 NIFI-6753: Fixed bug where all values being provided to the CSV Writer were String objects, which resulted in the CSV Writer improperly quoting numeric values when the schema indicates that the value is a number. Now, we will only convert the value to a String if the value is not a Number and/or the schema does not indicate a numeric ty type Signed-off-by: Matthew Burgess <mattyb...@apache.org> This closes #3797 --- .../java/org/apache/nifi/csv/WriteCSVResult.java | 40 +++++++++++++++++----- .../org/apache/nifi/csv/TestWriteCSVResult.java | 37 ++++++++++++++++++++ 2 files changed, 68 insertions(+), 9 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/WriteCSVResult.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/WriteCSVResult.java index 020b7b1..6526ebe 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/WriteCSVResult.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/WriteCSVResult.java @@ -17,14 +17,6 @@ package org.apache.nifi.csv; -import java.io.IOException; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.util.LinkedHashSet; -import java.util.Map; -import java.util.Optional; -import java.util.Set; - import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVPrinter; import org.apache.nifi.schema.access.SchemaAccessWriter; @@ -35,8 +27,17 @@ import org.apache.nifi.serialization.record.DataType; import org.apache.nifi.serialization.record.RawRecordWriter; import org.apache.nifi.serialization.record.Record; import org.apache.nifi.serialization.record.RecordField; +import org.apache.nifi.serialization.record.RecordFieldType; import org.apache.nifi.serialization.record.RecordSchema; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + public class WriteCSVResult extends AbstractRecordSetWriter implements RecordSetWriter, RawRecordWriter { private final RecordSchema recordSchema; private final SchemaAccessWriter schemaWriter; @@ -142,13 +143,34 @@ public class WriteCSVResult extends AbstractRecordSetWriter implements RecordSet int i = 0; for (final RecordField recordField : recordSchema.getFields()) { - fieldValues[i++] = record.getAsString(recordField, getFormat(recordField)); + fieldValues[i++] = getFieldValue(record, recordField); } printer.printRecord(fieldValues); return schemaWriter.getAttributes(recordSchema); } + private Object getFieldValue(final Record record, final RecordField recordField) { + final RecordFieldType fieldType = recordField.getDataType().getFieldType(); + + switch (fieldType) { + case BIGINT: + case BYTE: + case DOUBLE: + case FLOAT: + case LONG: + case INT: + case SHORT: + final Object value = record.getValue(recordField); + if (value instanceof Number) { + return value; + } + break; + } + + return record.getAsString(recordField, getFormat(recordField)); + } + @Override public WriteResult writeRawRecord(final Record record) throws IOException { // If we are not writing an active record set, then we need to ensure that we write the diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestWriteCSVResult.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestWriteCSVResult.java index 1cfaafd..bb3d0fe 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestWriteCSVResult.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestWriteCSVResult.java @@ -52,6 +52,43 @@ import static org.junit.Assert.assertEquals; public class TestWriteCSVResult { @Test + public void testNumbersNotQuoted() throws IOException { + final Map<String, Object> values = new HashMap<>(); + values.put("name", "John Doe"); + values.put("age", 30); + + final List<RecordField> schemaFields = new ArrayList<>(); + schemaFields.add(new RecordField("name", RecordFieldType.STRING.getDataType())); + schemaFields.add(new RecordField("age", RecordFieldType.INT.getDataType())); + + final RecordSchema schema = new SimpleRecordSchema(schemaFields); + final Record record = new MapRecord(schema, values); + + // Test with Non-Numeric Quote Mode + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + CSVFormat csvFormat = CSVFormat.DEFAULT.withQuoteMode(QuoteMode.NON_NUMERIC).withRecordSeparator("\n"); + try (final WriteCSVResult result = new WriteCSVResult(csvFormat, schema, new SchemaNameAsAttribute(), baos, + RecordFieldType.DATE.getDefaultFormat(), RecordFieldType.TIME.getDefaultFormat(), RecordFieldType.TIMESTAMP.getDefaultFormat(), true, "UTF-8")) { + result.writeRecord(record); + } + + String output = baos.toString(); + assertEquals("\"name\",\"age\"\n\"John Doe\",30\n", output); + + baos.reset(); + + // Test with MINIMAL Quote Mode + csvFormat = CSVFormat.DEFAULT.withQuoteMode(QuoteMode.MINIMAL).withRecordSeparator("\n"); + try (final WriteCSVResult result = new WriteCSVResult(csvFormat, schema, new SchemaNameAsAttribute(), baos, + RecordFieldType.DATE.getDefaultFormat(), RecordFieldType.TIME.getDefaultFormat(), RecordFieldType.TIMESTAMP.getDefaultFormat(), true, "UTF-8")) { + result.writeRecord(record); + } + + output = baos.toString(); + assertEquals("name,age\nJohn Doe,30\n", output); + } + + @Test public void testDataTypes() throws IOException { final CSVFormat csvFormat = CSVFormat.DEFAULT.withQuoteMode(QuoteMode.ALL).withRecordSeparator("\n");