I'm new to Camel, but have followed advice on various fora to produce the
below route:
Picks up a csv file, splits into lines and unmarshals each line into a Map
to be inserted
into a database, as named parameters.
The route works (database gets updated), when I use a subset of data (up to
30,000 lines).
However, my full file contains 115,000 lines. That file produces the
stacktrace below.
How must this route be changed to be able to accommodate the full 115,000
line file? (I'm trying to stick to a XML DSL solution)
<route>
<from uri="direct:dbload"/>
<pollEnrich>
<simple>file:{{ftp.unzipFolder}}/?fileName={{ftp.unzipName}}</simple>
</pollEnrich>
<split parallelProcessing="true" streaming="true">
<tokenize token="\r\n"/>
<filter id="_filter1">
<simple>${header.CamelSplitIndex} > 0</simple>
<pipeline id="_pipeline1">
<unmarshal>
<csv delimiter="," formatName="EXCEL"
useMaps="true">
<header>id</header>
</csv>
</unmarshal>
</pipeline>
</filter>
<transform>
<simple>${body[0]}</simple>
</transform>
<to id="_to3"
uri="sql:classpath:/assembly/etc/db-insert.sql?dataSource=dataSource"/>
</split>
</route>
sh-4.2$ ps -ef | grep -ai jvm
jboss 1 0 26 17:46 ? 00:00:38
/usr/lib/jvm/java-1.8.0/bin/java -DXms=512M -DXmx=1024M
-XX:ParallelGCThreads=1 -XX:ConcGCThreads=1
-Djava.util.concurrent.ForkJoinPool.common.parall
Stacktrace
---------------------------------------------------------------------------------------------------------------------------------------
org.apache.camel.CamelExecutionException: Exception occurred during
execution on the exchange:
Exchange[ID-medpages-68-q75qr-41525-1495388795330-0-36]
at
org.apache.camel.util.ObjectHelper.wrapCamelExecutionException(ObjectHelper.java:1706)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.impl.DefaultExchange.setException(DefaultExchange.java:348)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.processor.TransformProcessor.process(TransformProcessor.java:83)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.management.InstrumentationProcessor.process(InstrumentationProcessor.java:77)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:95)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.builder.NoErrorHandlerBuilder$1.process(NoErrorHandlerBuilder.java:40)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:196)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.processor.Pipeline.process(Pipeline.java:121)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.processor.Pipeline.process(Pipeline.java:83)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:95)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.builder.NoErrorHandlerBuilder$1.process(NoErrorHandlerBuilder.java:40)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:196)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.util.AsyncProcessorHelper.process(AsyncProcessorHelper.java:109)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.processor.MulticastProcessor.doProcessParallel(MulticastProcessor.java:827)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.processor.MulticastProcessor.access$200(MulticastProcessor.java:85)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.processor.MulticastProcessor$1.call(MulticastProcessor.java:320)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.processor.MulticastProcessor$1.call(MulticastProcessor.java:305)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at java.util.concurrent.FutureTask.run(FutureTask.java:266)[:1.8.0_131]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)[:1.8.0_131]
at java.util.concurrent.FutureTask.run(FutureTask.java:266)[:1.8.0_131]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)[:1.8.0_131]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)[:1.8.0_131]
at java.lang.Thread.run(Thread.java:748)[:1.8.0_131]
Caused by: java.lang.OutOfMemoryError: Java heap space
at java.util.Arrays.copyOf(Arrays.java:3332)[:1.8.0_131]
at
java.lang.AbstractStringBuilder.ensureCapacityInternal(AbstractStringBuilder.java:124)[:1.8.0_131]
at
java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:448)[:1.8.0_131]
at java.lang.StringBuilder.append(StringBuilder.java:136)[:1.8.0_131]
at
org.apache.camel.language.bean.BeanExpression$OgnlInvokeProcessor.lookupResult(BeanExpression.java:391)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.language.bean.BeanExpression$OgnlInvokeProcessor.process(BeanExpression.java:329)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.language.bean.BeanExpression.evaluate(BeanExpression.java:114)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.language.bean.BeanExpression.evaluate(BeanExpression.java:138)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.model.language.ExpressionDefinition.evaluate(ExpressionDefinition.java:126)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.model.language.ExpressionDefinition.evaluate(ExpressionDefinition.java:118)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.builder.ExpressionBuilder$34.evaluate(ExpressionBuilder.java:852)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.support.ExpressionAdapter.evaluate(ExpressionAdapter.java:36)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.builder.SimpleBuilder.evaluate(SimpleBuilder.java:92)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.processor.TransformProcessor.process(TransformProcessor.java:50)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.management.InstrumentationProcessor.process(InstrumentationProcessor.java:77)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:95)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.builder.NoErrorHandlerBuilder$1.process(NoErrorHandlerBuilder.java:40)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:196)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.processor.Pipeline.process(Pipeline.java:121)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.processor.Pipeline.process(Pipeline.java:83)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:95)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.builder.NoErrorHandlerBuilder$1.process(NoErrorHandlerBuilder.java:40)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:196)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.util.AsyncProcessorHelper.process(AsyncProcessorHelper.java:109)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.processor.MulticastProcessor.doProcessParallel(MulticastProcessor.java:827)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.processor.MulticastProcessor.access$200(MulticastProcessor.java:85)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.processor.MulticastProcessor$1.call(MulticastProcessor.java:320)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at
org.apache.camel.processor.MulticastProcessor$1.call(MulticastProcessor.java:305)[101:org.apache.camel.camel-core:2.17.0.redhat-630224]
at java.util.concurrent.FutureTask.run(FutureTask.java:266)[:1.8.0_131]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)[:1.8.0_131]
at java.util.concurrent.FutureTask.run(FutureTask.java:266)[:1.8.0_131]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)[:1.8.0_131]
--
View this message in context:
http://camel.465427.n5.nabble.com/61M-csv-file-cause-Heap-error-on-JVM-with-1-GB-limit-tp5799976.html
Sent from the Camel - Users mailing list archive at Nabble.com.
