Repository: camel Updated Branches: refs/heads/master 05f13c00a -> 0e72f475f
CAMEL-9476: added possibility to ignore too short lines for fixed layout Project: http://git-wip-us.apache.org/repos/asf/camel/repo Commit: http://git-wip-us.apache.org/repos/asf/camel/commit/0e72f475 Tree: http://git-wip-us.apache.org/repos/asf/camel/tree/0e72f475 Diff: http://git-wip-us.apache.org/repos/asf/camel/diff/0e72f475 Branch: refs/heads/master Commit: 0e72f475f80a9699fe49004c431e1ce4e360c35d Parents: 05f13c0 Author: Arno Noordover <anoordo...@users.noreply.github.com> Authored: Tue Jun 28 22:00:39 2016 +0200 Committer: Arno Noordover <anoordo...@users.noreply.github.com> Committed: Tue Jun 28 22:00:39 2016 +0200 ---------------------------------------------------------------------- .../bindy/BindyFixedLengthFactory.java | 61 +++++++++++- .../bindy/annotation/FixedLengthRecord.java | 4 + .../bindy/fixed/BindyFixedLengthDataFormat.java | 73 +++++++++------ .../fixed/BindyPaddingAndTrimmingTest.java | 97 ++++++++++++++++++++ ...impleFixedLengthUnmarshallTrimFieldTest.java | 6 +- 5 files changed, 209 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/camel/blob/0e72f475/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/BindyFixedLengthFactory.java ---------------------------------------------------------------------- diff --git a/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/BindyFixedLengthFactory.java b/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/BindyFixedLengthFactory.java index c517852..302a2f7 100644 --- a/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/BindyFixedLengthFactory.java +++ b/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/BindyFixedLengthFactory.java @@ -64,6 +64,7 @@ public class BindyFixedLengthFactory extends BindyAbstractFactory implements Bin private char paddingChar; private int recordLength; private boolean ignoreTrailingChars; + private boolean ignoreMissingChars; private Class<?> header; private Class<?> footer; @@ -201,7 +202,15 @@ public class BindyFixedLengthFactory extends BindyAbstractFactory implements Bin } if (length > 0) { - token = record.substring(offset - 1, offset + length - 1); + if (record.length() < offset) { + token = ""; + } else { + int endIndex = offset + length - 1; + if (endIndex > record.length()) { + endIndex = record.length(); + } + token = record.substring(offset - 1, endIndex); + } offset += length; } else if (!delimiter.equals("")) { String tempToken = record.substring(offset - 1, record.length()); @@ -214,7 +223,8 @@ public class BindyFixedLengthFactory extends BindyAbstractFactory implements Bin } if (dataField.trim()) { - token = token.trim(); + token = trim(token, dataField, paddingChar); + //token = token.trim(); } // Check mandatory field @@ -252,7 +262,10 @@ public class BindyFixedLengthFactory extends BindyAbstractFactory implements Bin // format the data received Object value = null; - if (!token.equals("")) { + if ("".equals(token)) { + token = dataField.defaultValue(); + } + if (!"".equals(token)) { try { value = format.parse(token); } catch (FormatException ie) { @@ -287,6 +300,38 @@ public class BindyFixedLengthFactory extends BindyAbstractFactory implements Bin } + private String trim(String token, DataField dataField, char paddingChar) { + char myPaddingChar = dataField.paddingChar(); + if (dataField.paddingChar() == 0) { + myPaddingChar = paddingChar; + } + if ("R".equals(dataField.align())) { + return leftTrim(token, myPaddingChar); + } else { + return rightTrim(token, myPaddingChar); + } + } + + private String rightTrim(String token, char myPaddingChar) { + StringBuilder sb = new StringBuilder(token); + + while (sb.length() > 0 && myPaddingChar == sb.charAt(sb.length() - 1)) { + sb.deleteCharAt(sb.length() - 1); + } + + return sb.toString(); + } + + private String leftTrim(String token, char myPaddingChar) { + StringBuilder sb = new StringBuilder(token); + + while (sb.length() > 0 && myPaddingChar == (sb.charAt(0))) { + sb.deleteCharAt(0); + } + + return sb.toString(); + } + @Override public String unbind(Map<String, Object> model) throws Exception { @@ -516,6 +561,9 @@ public class BindyFixedLengthFactory extends BindyAbstractFactory implements Bin // Get flag for ignore trailing characters ignoreTrailingChars = record.ignoreTrailingChars(); LOG.debug("Ignore trailing chars: {}", ignoreTrailingChars); + + ignoreMissingChars = record.ignoreMissingChars(); + LOG.debug("Enable ignore missing chars: {}", ignoreMissingChars); } } @@ -613,4 +661,11 @@ public class BindyFixedLengthFactory extends BindyAbstractFactory implements Bin return this.ignoreTrailingChars; } + /** + * Flag indicating whether too short lines are ignored + */ + public boolean isIgnoreMissingChars() { + return ignoreMissingChars; + } + } http://git-wip-us.apache.org/repos/asf/camel/blob/0e72f475/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/annotation/FixedLengthRecord.java ---------------------------------------------------------------------- diff --git a/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/annotation/FixedLengthRecord.java b/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/annotation/FixedLengthRecord.java index eca6fd1..ba1e25b 100644 --- a/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/annotation/FixedLengthRecord.java +++ b/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/annotation/FixedLengthRecord.java @@ -81,4 +81,8 @@ public @interface FixedLengthRecord { */ boolean ignoreTrailingChars() default false; + /** + * Indicates whether too short lines will be ignored + */ + boolean ignoreMissingChars() default false; } http://git-wip-us.apache.org/repos/asf/camel/blob/0e72f475/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/fixed/BindyFixedLengthDataFormat.java ---------------------------------------------------------------------- diff --git a/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/fixed/BindyFixedLengthDataFormat.java b/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/fixed/BindyFixedLengthDataFormat.java index 22efd73..23fce52 100644 --- a/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/fixed/BindyFixedLengthDataFormat.java +++ b/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/fixed/BindyFixedLengthDataFormat.java @@ -45,15 +45,15 @@ import org.slf4j.LoggerFactory; * {@link DataFormat}) using Bindy to marshal to and from Fixed Length */ public class BindyFixedLengthDataFormat extends BindyAbstractDataFormat { - + public static final String CAMEL_BINDY_FIXED_LENGTH_HEADER = "CamelBindyFixedLengthHeader"; public static final String CAMEL_BINDY_FIXED_LENGTH_FOOTER = "CamelBindyFixedLengthFooter"; private static final Logger LOG = LoggerFactory.getLogger(BindyFixedLengthDataFormat.class); - + private BindyFixedLengthFactory headerFactory; private BindyFixedLengthFactory footerFactory; - + public BindyFixedLengthDataFormat() { } @@ -65,7 +65,7 @@ public class BindyFixedLengthDataFormat extends BindyAbstractDataFormat { public String getDataFormatName() { return "bindy-fixed"; } - + @SuppressWarnings("unchecked") public void marshal(Exchange exchange, Object body, OutputStream outputStream) throws Exception { BindyFixedLengthFactory factory = (BindyFixedLengthFactory) getFactory(); @@ -92,13 +92,13 @@ public class BindyFixedLengthDataFormat extends BindyAbstractDataFormat { // cast to the expected type models = (List<Map<String, Object>>) body; } - + // add the header if it is in the exchange header Map<String, Object> headerRow = (Map<String, Object>) exchange.getIn().getHeader(CAMEL_BINDY_FIXED_LENGTH_HEADER); if (headerRow != null) { models.add(0, headerRow); } - + // add the footer if it is in the exchange header Map<String, Object> footerRow = (Map<String, Object>) exchange.getIn().getHeader(CAMEL_BINDY_FIXED_LENGTH_FOOTER); if (footerRow != null) { @@ -109,18 +109,18 @@ public class BindyFixedLengthDataFormat extends BindyAbstractDataFormat { for (Map<String, Object> model : models) { row++; String result = null; - + if (row == 1 && headerFactory != null) { // marshal the first row as a header if the models match Set<String> modelClassNames = model.keySet(); // only use the header factory if the row is the header if (headerFactory.supportsModel(modelClassNames)) { - if (factory.skipHeader()) { + if (factory.skipHeader()) { LOG.info("Skipping marshal of header row; 'skipHeader=true'"); continue; } else { result = headerFactory.unbind(model); - } + } } } else if (row == models.size() && footerFactory != null) { // marshal the last row as a footer if the models match @@ -135,12 +135,12 @@ public class BindyFixedLengthDataFormat extends BindyAbstractDataFormat { } } } - + if (result == null) { // marshal as a normal / default row result = factory.unbind(model); } - + byte[] bytes = exchange.getContext().getTypeConverter().convertTo(byte[].class, exchange, result); outputStream.write(bytes); @@ -178,7 +178,7 @@ public class BindyFixedLengthDataFormat extends BindyAbstractDataFormat { public Object unmarshal(Exchange exchange, InputStream inputStream) throws Exception { BindyFixedLengthFactory factory = (BindyFixedLengthFactory) getFactory(); ObjectHelper.notNull(factory, "not instantiated"); - + // List of Pojos List<Map<String, Object>> models = new ArrayList<Map<String, Object>>(); @@ -196,10 +196,10 @@ public class BindyFixedLengthDataFormat extends BindyAbstractDataFormat { // Parse the header if it exists if (scanner.hasNextLine() && factory.hasHeader()) { - + // Read the line (should not trim as its fixed length) String line = getNextNonEmptyLine(scanner, count); - + if (!factory.skipHeader()) { Map<String, Object> headerObjMap = createModel(headerFactory, line, count.intValue()); exchange.getOut().setHeader(CAMEL_BINDY_FIXED_LENGTH_HEADER, headerObjMap); @@ -215,7 +215,7 @@ public class BindyFixedLengthDataFormat extends BindyAbstractDataFormat { // Parse the main file content while (thisLine != null && nextLine != null) { - + model = createModel(factory, thisLine, count.intValue()); // Add objects graph to the list @@ -224,7 +224,7 @@ public class BindyFixedLengthDataFormat extends BindyAbstractDataFormat { thisLine = nextLine; nextLine = getNextNonEmptyLine(scanner, count); } - + // this line should be the last non-empty line from the file // optionally parse the line as a footer if (thisLine != null) { @@ -260,7 +260,7 @@ public class BindyFixedLengthDataFormat extends BindyAbstractDataFormat { count.incrementAndGet(); line = scanner.nextLine(); } - + if (ObjectHelper.isEmpty(line)) { return null; } else { @@ -269,47 +269,68 @@ public class BindyFixedLengthDataFormat extends BindyAbstractDataFormat { } protected Map<String, Object> createModel(BindyFixedLengthFactory factory, String line, int count) throws Exception { + String myLine = line; + // Check if the record length corresponds to the parameter // provided in the @FixedLengthRecord if (factory.recordLength() > 0) { - if ((line.length() < factory.recordLength()) || (line.length() > factory.recordLength())) { - throw new java.lang.IllegalArgumentException("Size of the record: " + line.length() + if (isPaddingNeededAndEnable(factory, myLine)) { + //myLine = rightPad(myLine, factory.recordLength()); + } + if (isTrimmingNeededAndEnabled(factory, myLine)) { + myLine = myLine.substring(0, factory.recordLength()); + } + if ((myLine.length() < factory.recordLength() + && !factory.isIgnoreMissingChars()) || (myLine.length() > factory.recordLength())) { + throw new java.lang.IllegalArgumentException("Size of the record: " + myLine.length() + " is not equal to the value provided in the model: " + factory.recordLength()); } } // Create POJO where Fixed data will be stored Map<String, Object> model = factory.factory(); - + // Bind data from Fixed record with model classes - factory.bind(line, model, count); + factory.bind(myLine, model, count); // Link objects together factory.link(model); - + LOG.debug("Graph of objects created: {}", model); return model; } + private boolean isTrimmingNeededAndEnabled(BindyFixedLengthFactory factory, String myLine) { + return factory.isIgnoreTrailingChars() && myLine.length() > factory.recordLength(); + } + + private String rightPad(String myLine, int length) { + return String.format("%1$-" + length + "s", myLine); + } + + private boolean isPaddingNeededAndEnable(BindyFixedLengthFactory factory, String myLine) { + return myLine.length() < factory.recordLength() && factory.isIgnoreMissingChars(); + } + @Override protected BindyAbstractFactory createModelFactory(FormatFactory formatFactory) throws Exception { BindyFixedLengthFactory factory = new BindyFixedLengthFactory(getClassType()); factory.setFormatFactory(formatFactory); - + // Optionally initialize the header factory... using header model classes if (factory.hasHeader()) { this.headerFactory = new BindyFixedLengthFactory(factory.header()); this.headerFactory.setFormatFactory(formatFactory); } - + // Optionally initialize the footer factory... using footer model classes if (factory.hasFooter()) { this.footerFactory = new BindyFixedLengthFactory(factory.footer()); this.footerFactory.setFormatFactory(formatFactory); } - + return factory; } - + } http://git-wip-us.apache.org/repos/asf/camel/blob/0e72f475/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/fixed/BindyPaddingAndTrimmingTest.java ---------------------------------------------------------------------- diff --git a/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/fixed/BindyPaddingAndTrimmingTest.java b/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/fixed/BindyPaddingAndTrimmingTest.java new file mode 100644 index 0000000..9e3e174 --- /dev/null +++ b/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/fixed/BindyPaddingAndTrimmingTest.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dataformat.bindy.fixed; + +import org.apache.camel.EndpointInject; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.component.mock.MockEndpoint; +import org.apache.camel.dataformat.bindy.annotation.DataField; +import org.apache.camel.dataformat.bindy.annotation.FixedLengthRecord; +import org.apache.camel.model.dataformat.BindyType; +import org.apache.camel.test.junit4.CamelTestSupport; +import org.hamcrest.core.Is; +import org.junit.Test; + +import static org.hamcrest.core.IsNull.nullValue; + +public class BindyPaddingAndTrimmingTest extends CamelTestSupport { + + private static final String URI_DIRECT_UNMARSHAL = "direct:unmarshall"; + private static final String URI_MOCK_UNMARSHAL_RESULT = "mock:unmarshal_result"; + + @EndpointInject(uri = URI_MOCK_UNMARSHAL_RESULT) + private MockEndpoint unmarhsalResult; + + @Override + protected RouteBuilder createRouteBuilder() throws Exception { + return new RouteBuilder() { + @Override + public void configure() throws Exception { + from(URI_DIRECT_UNMARSHAL) + .unmarshal().bindy(BindyType.Fixed, MyBindyModel.class) + .to(URI_MOCK_UNMARSHAL_RESULT); + } + }; + } + + @Test + public void testUnmarshal() throws Exception { + unmarhsalResult.expectedMessageCount(1); + template.sendBody(URI_DIRECT_UNMARSHAL, "foo \r\n"); + + unmarhsalResult.assertIsSatisfied(); + MyBindyModel myBindyModel = unmarhsalResult.getReceivedExchanges().get(0).getIn().getBody(MyBindyModel.class); + assertEquals("foo ", myBindyModel.foo); + assertThat(myBindyModel.bar, Is.is(nullValue())); + } + + @Test + public void testUnmarshalTooLong() throws Exception { + unmarhsalResult.expectedMessageCount(1); + template.sendBody(URI_DIRECT_UNMARSHAL, "foo bar \r\n"); + + unmarhsalResult.assertIsSatisfied(); + MyBindyModel myBindyModel = unmarhsalResult.getReceivedExchanges().get(0).getIn().getBody(MyBindyModel.class); + assertEquals("foo ", myBindyModel.foo); + + } + + @FixedLengthRecord(length = 10, ignoreMissingChars = true, ignoreTrailingChars = true) + public static class MyBindyModel { + @DataField(pos = 0, length = 5) + String foo; + + @DataField(pos = 5, length = 5) + String bar; + + public String getFoo() { + return foo; + } + + public void setFoo(String foo) { + this.foo = foo; + } + + public String getBar() { + return bar; + } + + public void setBar(String bar) { + this.bar = bar; + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/camel/blob/0e72f475/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/fixed/unmarshall/simple/trimfield/BindySimpleFixedLengthUnmarshallTrimFieldTest.java ---------------------------------------------------------------------- diff --git a/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/fixed/unmarshall/simple/trimfield/BindySimpleFixedLengthUnmarshallTrimFieldTest.java b/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/fixed/unmarshall/simple/trimfield/BindySimpleFixedLengthUnmarshallTrimFieldTest.java index e26dcae..452d951 100644 --- a/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/fixed/unmarshall/simple/trimfield/BindySimpleFixedLengthUnmarshallTrimFieldTest.java +++ b/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/fixed/unmarshall/simple/trimfield/BindySimpleFixedLengthUnmarshallTrimFieldTest.java @@ -51,7 +51,7 @@ public class BindySimpleFixedLengthUnmarshallTrimFieldTest extends AbstractJUnit @DirtiesContext public void testUnMarshallMessage() throws Exception { - expected = "10A9 PaulineM ISINXD12345678BUYShare000002500.45USD01-08-2009 Hello "; + expected = "10A9 PaulineM ISINXD12345678BUYShare000002500.45USD01-08-2009 Hello###"; template.sendBody(expected); @@ -65,7 +65,7 @@ public class BindySimpleFixedLengthUnmarshallTrimFieldTest extends AbstractJUnit // the field is not trimmed Assert.assertEquals("Pauline", order.getFirstName()); Assert.assertEquals("M ", order.getLastName()); // no trim - Assert.assertEquals("Hello", order.getComment()); + Assert.assertEquals(" Hello", order.getComment()); } public static class ContextConfig extends RouteBuilder { @@ -113,7 +113,7 @@ public class BindySimpleFixedLengthUnmarshallTrimFieldTest extends AbstractJUnit @DataField(pos = 56, length = 10, pattern = "dd-MM-yyyy") private Date orderDate; - @DataField(pos = 66, length = 10, trim = true) + @DataField(pos = 66, length = 10, trim = true, align = "L", paddingChar = '#') private String comment; public int getOrderNr() {