Author: cmueller Date: Sat Nov 10 11:30:17 2012 New Revision: 1407765 URL: http://svn.apache.org/viewvc?rev=1407765&view=rev Log: CAMEL-5785: Add support for handling long/short lines in Fixed Width Files in the Flatpack DataFormat Thanks to Chris Geer for the patch
Modified: camel/branches/camel-2.9.x/components/camel-flatpack/src/main/java/org/apache/camel/component/flatpack/FlatpackDataFormat.java camel/branches/camel-2.9.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowLongTest.java camel/branches/camel-2.9.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowShortAndLongTest.java camel/branches/camel-2.9.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowShortTest.java camel/branches/camel-2.9.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowLongTest-context.xml camel/branches/camel-2.9.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowShortAndLongTest-context.xml camel/branches/camel-2.9.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowShortTest-context.xml Modified: camel/branches/camel-2.9.x/components/camel-flatpack/src/main/java/org/apache/camel/component/flatpack/FlatpackDataFormat.java URL: http://svn.apache.org/viewvc/camel/branches/camel-2.9.x/components/camel-flatpack/src/main/java/org/apache/camel/component/flatpack/FlatpackDataFormat.java?rev=1407765&r1=1407764&r2=1407765&view=diff ============================================================================== --- camel/branches/camel-2.9.x/components/camel-flatpack/src/main/java/org/apache/camel/component/flatpack/FlatpackDataFormat.java (original) +++ camel/branches/camel-2.9.x/components/camel-flatpack/src/main/java/org/apache/camel/component/flatpack/FlatpackDataFormat.java Sat Nov 10 11:30:17 2012 @@ -61,6 +61,8 @@ public class FlatpackDataFormat implemen private char textQualifier = '"'; private boolean ignoreFirstRecord = true; private boolean fixed; + private boolean allowShortLines; + private boolean ignoreExtraColumns; private String definition; @SuppressWarnings("unchecked") @@ -158,6 +160,28 @@ public class FlatpackDataFormat implemen this.parserFactory = parserFactory; } + public boolean isAllowShortLines() { + return this.allowShortLines; + } + + /** + * Allows for lines to be shorter than expected and ignores the extra characters + */ + public void setAllowShortLines(boolean allowShortLines) { + this.allowShortLines = allowShortLines; + } + + /** + * Allows for lines to be longer than expected and ignores the extra characters + */ + public void setIgnoreExtraColumns(boolean ignoreExtraColumns) { + this.ignoreExtraColumns = ignoreExtraColumns; + } + + public boolean isIgnoreExtraColumns() { + return ignoreExtraColumns; + } + // Implementation methods //------------------------------------------------------------------------- @@ -165,7 +189,16 @@ public class FlatpackDataFormat implemen if (isFixed()) { InputStream is = ResourceHelper.resolveMandatoryResourceAsInputStream(exchange.getContext().getClassResolver(), getDefinition()); InputStreamReader reader = new InputStreamReader(is, IOHelper.getCharsetName(exchange)); - return getParserFactory().newFixedLengthParser(reader, bodyReader); + Parser parser = getParserFactory().newFixedLengthParser(reader, bodyReader); + if (allowShortLines) { + parser.setHandlingShortLines(true); + parser.setIgnoreParseWarnings(true); + } + if (ignoreExtraColumns) { + parser.setIgnoreExtraColumns(true); + parser.setIgnoreParseWarnings(true); + } + return parser; } else { if (ObjectHelper.isEmpty(getDefinition())) { return getParserFactory().newDelimitedParser(bodyReader, delimiter, textQualifier); Modified: camel/branches/camel-2.9.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowLongTest.java URL: http://svn.apache.org/viewvc/camel/branches/camel-2.9.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowLongTest.java?rev=1407765&r1=1407764&r2=1407765&view=diff ============================================================================== --- camel/branches/camel-2.9.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowLongTest.java (original) +++ camel/branches/camel-2.9.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowLongTest.java Sat Nov 10 11:30:17 2012 @@ -44,6 +44,9 @@ public class FixedLengthAllowLongTest ex @EndpointInject(uri = "mock:results") protected MockEndpoint results; + @EndpointInject(uri = "mock:results-df") + protected MockEndpoint resultsdf; + protected String[] expectedFirstName = {"JOHN-LONG", "JIMMY-LONG", "JANE-LONG", "FRED-LONG"}; @Test @@ -63,4 +66,18 @@ public class FixedLengthAllowLongTest ex counter++; } } + + @Test + public void testFlatpackDataFormat() throws Exception { + resultsdf.expectedMessageCount(1); + resultsdf.assertIsSatisfied(); + + Exchange exchange = resultsdf.getReceivedExchanges().get(0); + DataSetList data = exchange.getIn().getBody(DataSetList.class); + int counter = 0; + for (Map<String, Object> map : data) { + assertEquals("FIRSTNAME", expectedFirstName[counter], map.get("FIRSTNAME")); + counter++; + } + } } Modified: camel/branches/camel-2.9.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowShortAndLongTest.java URL: http://svn.apache.org/viewvc/camel/branches/camel-2.9.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowShortAndLongTest.java?rev=1407765&r1=1407764&r2=1407765&view=diff ============================================================================== --- camel/branches/camel-2.9.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowShortAndLongTest.java (original) +++ camel/branches/camel-2.9.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowShortAndLongTest.java Sat Nov 10 11:30:17 2012 @@ -44,10 +44,13 @@ public class FixedLengthAllowShortAndLon @EndpointInject(uri = "mock:results") protected MockEndpoint results; + @EndpointInject(uri = "mock:results-df") + protected MockEndpoint resultsdf; + protected String[] expectedFirstName = {"JOHN-LONG", "JIMMY-SHORT", "JANE-LONG", "FRED-NORMAL"}; @Test - public void testCamel() throws Exception { + public void testFlatpack() throws Exception { results.expectedMessageCount(4); results.assertIsSatisfied(); @@ -63,4 +66,18 @@ public class FixedLengthAllowShortAndLon counter++; } } + + @Test + public void testFlatpackDataFormat() throws Exception { + resultsdf.expectedMessageCount(1); + resultsdf.assertIsSatisfied(); + + Exchange exchange = resultsdf.getReceivedExchanges().get(0); + DataSetList data = exchange.getIn().getBody(DataSetList.class); + int counter = 0; + for (Map<String, Object> map : data) { + assertEquals("FIRSTNAME", expectedFirstName[counter], map.get("FIRSTNAME")); + counter++; + } + } } Modified: camel/branches/camel-2.9.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowShortTest.java URL: http://svn.apache.org/viewvc/camel/branches/camel-2.9.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowShortTest.java?rev=1407765&r1=1407764&r2=1407765&view=diff ============================================================================== --- camel/branches/camel-2.9.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowShortTest.java (original) +++ camel/branches/camel-2.9.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowShortTest.java Sat Nov 10 11:30:17 2012 @@ -44,6 +44,9 @@ public class FixedLengthAllowShortTest e @EndpointInject(uri = "mock:results") protected MockEndpoint results; + @EndpointInject(uri = "mock:results-df") + protected MockEndpoint resultsdf; + protected String[] expectedFirstName = {"JOHN-SHORT", "JIMMY-SHORT", "JANE-SHORT", "FRED-SHORT"}; @Test @@ -63,4 +66,18 @@ public class FixedLengthAllowShortTest e counter++; } } + + @Test + public void testFlatpackDataFormat() throws Exception { + resultsdf.expectedMessageCount(1); + resultsdf.assertIsSatisfied(); + + Exchange exchange = resultsdf.getReceivedExchanges().get(0); + DataSetList data = exchange.getIn().getBody(DataSetList.class); + int counter = 0; + for (Map<String, Object> map : data) { + assertEquals("FIRSTNAME", expectedFirstName[counter], map.get("FIRSTNAME")); + counter++; + } + } } Modified: camel/branches/camel-2.9.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowLongTest-context.xml URL: http://svn.apache.org/viewvc/camel/branches/camel-2.9.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowLongTest-context.xml?rev=1407765&r1=1407764&r2=1407765&view=diff ============================================================================== --- camel/branches/camel-2.9.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowLongTest-context.xml (original) +++ camel/branches/camel-2.9.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowLongTest-context.xml Sat Nov 10 11:30:17 2012 @@ -22,11 +22,20 @@ http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd "> + <bean id="df" class="org.apache.camel.component.flatpack.FlatpackDataFormat"> + <property name="definition" value="PEOPLE-FixedLength.pzmap.xml"/> + <property name="fixed" value="true"/> + <property name="ignoreExtraColumns" value="true"/> + </bean> + <!-- START SNIPPET: example --> <camelContext xmlns="http://camel.apache.org/schema/spring"> <route> <from uri="file://src/test/data/fixedLong?noop=true"/> - <to uri="flatpack:fixed:PEOPLE-FixedLength.pzmap.xml?ignoreExtraColumns=true"/> + <multicast> + <to uri="flatpack:fixed:PEOPLE-FixedLength.pzmap.xml?ignoreExtraColumns=true"/> + <to uri="direct:df"/> + </multicast> </route> <route> @@ -34,6 +43,13 @@ <convertBodyTo type="java.util.Map"/> <to uri="mock:results"/> </route> + + <route> + <from uri="direct:df"/> + <unmarshal ref="df"/> + <to uri="mock:results-df"/> + </route> + </camelContext> <!-- END SNIPPET: example --> Modified: camel/branches/camel-2.9.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowShortAndLongTest-context.xml URL: http://svn.apache.org/viewvc/camel/branches/camel-2.9.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowShortAndLongTest-context.xml?rev=1407765&r1=1407764&r2=1407765&view=diff ============================================================================== --- camel/branches/camel-2.9.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowShortAndLongTest-context.xml (original) +++ camel/branches/camel-2.9.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowShortAndLongTest-context.xml Sat Nov 10 11:30:17 2012 @@ -22,11 +22,21 @@ http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd "> + <bean id="df" class="org.apache.camel.component.flatpack.FlatpackDataFormat"> + <property name="definition" value="PEOPLE-FixedLength.pzmap.xml"/> + <property name="fixed" value="true"/> + <property name="ignoreExtraColumns" value="true"/> + <property name="allowShortLines" value="true"/> + </bean> + <!-- START SNIPPET: example --> <camelContext xmlns="http://camel.apache.org/schema/spring"> <route> <from uri="file://src/test/data/fixedMixed?noop=true"/> - <to uri="flatpack:fixed:PEOPLE-FixedLength.pzmap.xml?ignoreExtraColumns=true&allowShortLines=true"/> + <multicast> + <to uri="flatpack:fixed:PEOPLE-FixedLength.pzmap.xml?ignoreExtraColumns=true&allowShortLines=true"/> + <to uri="direct:df"/> + </multicast> </route> <route> @@ -34,6 +44,12 @@ <convertBodyTo type="java.util.Map"/> <to uri="mock:results"/> </route> + + <route> + <from uri="direct:df"/> + <unmarshal ref="df"/> + <to uri="mock:results-df"/> + </route> </camelContext> <!-- END SNIPPET: example --> Modified: camel/branches/camel-2.9.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowShortTest-context.xml URL: http://svn.apache.org/viewvc/camel/branches/camel-2.9.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowShortTest-context.xml?rev=1407765&r1=1407764&r2=1407765&view=diff ============================================================================== --- camel/branches/camel-2.9.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowShortTest-context.xml (original) +++ camel/branches/camel-2.9.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowShortTest-context.xml Sat Nov 10 11:30:17 2012 @@ -22,11 +22,20 @@ http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd "> + <bean id="df" class="org.apache.camel.component.flatpack.FlatpackDataFormat"> + <property name="definition" value="PEOPLE-FixedLength.pzmap.xml"/> + <property name="fixed" value="true"/> + <property name="allowShortLines" value="true"/> + </bean> + <!-- START SNIPPET: example --> - <camelContext xmlns="http://camel.apache.org/schema/spring"> + <camelContext xmlns="http://camel.apache.org/schema/spring" trace="true"> <route> <from uri="file://src/test/data/fixedShort?noop=true"/> - <to uri="flatpack:fixed:PEOPLE-FixedLength.pzmap.xml?allowShortLines=true"/> + <multicast> + <to uri="flatpack:fixed:PEOPLE-FixedLength.pzmap.xml?allowShortLines=true"/> + <to uri="direct:df"/> + </multicast> </route> <route> @@ -34,6 +43,12 @@ <convertBodyTo type="java.util.Map"/> <to uri="mock:results"/> </route> + + <route> + <from uri="direct:df"/> + <unmarshal ref="df"/> + <to uri="mock:results-df"/> + </route> </camelContext> <!-- END SNIPPET: example -->