Author: davsclaus
Date: Sat Oct 29 12:37:21 2011
New Revision: 1194882
URL: http://svn.apache.org/viewvc?rev=1194882&view=rev
Log:
CAMEL-4589: Added tokenizer pair to split big XML files in streaming mode, with
low memory footprint.
Added:
camel/trunk/camel-core/src/main/java/org/apache/camel/processor/TokenPairExpressionIterator.java
camel/trunk/camel-core/src/test/java/org/apache/camel/language/TokenPairIteratorSplitChoicePerformanceTest.java
camel/trunk/camel-core/src/test/java/org/apache/camel/language/XPathSplitChoicePerformanceTest.java
camel/trunk/components/camel-saxon/src/test/java/org/apache/camel/builder/saxon/XPathSplitChoicePerformanceTest.java
(with props)
Modified:
camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionBuilder.java
camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionClause.java
camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionClauseSupport.java
camel/trunk/camel-core/src/main/java/org/apache/camel/language/tokenizer/TokenizeLanguage.java
camel/trunk/camel-core/src/main/java/org/apache/camel/model/language/TokenizerExpression.java
camel/trunk/camel-core/src/test/java/org/apache/camel/component/properties/PropertiesComponentTest.java
camel/trunk/camel-core/src/test/java/org/apache/camel/language/TokenizerTest.java
Modified:
camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionBuilder.java
URL:
http://svn.apache.org/viewvc/camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionBuilder.java?rev=1194882&r1=1194881&r2=1194882&view=diff
==============================================================================
---
camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionBuilder.java
(original)
+++
camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionBuilder.java
Sat Oct 29 12:37:21 2011
@@ -39,6 +39,7 @@ import org.apache.camel.component.bean.B
import org.apache.camel.component.properties.PropertiesComponent;
import org.apache.camel.language.bean.BeanLanguage;
import org.apache.camel.model.language.MethodCallExpression;
+import org.apache.camel.processor.TokenPairExpressionIterator;
import org.apache.camel.spi.Language;
import org.apache.camel.support.ExpressionAdapter;
import org.apache.camel.util.ExchangeHelper;
@@ -1024,6 +1025,13 @@ public final class ExpressionBuilder {
}
/**
+ * Returns an {@link TokenPairExpressionIterator} expression
+ */
+ public static Expression tokenizePairExpression(final String startToken,
final String endToken) {
+ return new TokenPairExpressionIterator(startToken, endToken);
+ }
+
+ /**
* Returns a tokenize expression which will tokenize the string with the
* given regex
*/
Modified:
camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionClause.java
URL:
http://svn.apache.org/viewvc/camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionClause.java?rev=1194882&r1=1194881&r2=1194882&view=diff
==============================================================================
---
camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionClause.java
(original)
+++
camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionClause.java
Sat Oct 29 12:37:21 2011
@@ -424,6 +424,17 @@ public class ExpressionClause<T> extends
}
/**
+ * Evaluates a token pair expression on the message body
+ *
+ * @param startToken the start token
+ * @param endToken the end token
+ * @return the builder to continue processing the DSL
+ */
+ public T tokenizePair(String startToken, String endToken) {
+ return delegate.tokenizePair(startToken, endToken);
+ }
+
+ /**
* Evaluates an <a href="http://camel.apache.org/xpath.html";>XPath
* expression</a>
*
Modified:
camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionClauseSupport.java
URL:
http://svn.apache.org/viewvc/camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionClauseSupport.java?rev=1194882&r1=1194881&r2=1194882&view=diff
==============================================================================
---
camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionClauseSupport.java
(original)
+++
camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionClauseSupport.java
Sat Oct 29 12:37:21 2011
@@ -471,6 +471,21 @@ public class ExpressionClauseSupport<T>
}
/**
+ * Evaluates a token pair expression on the message body
+ *
+ * @param startToken the start token
+ * @param endToken the end token
+ * @return the builder to continue processing the DSL
+ */
+ public T tokenizePair(String startToken, String endToken) {
+ TokenizerExpression expression = new TokenizerExpression();
+ expression.setToken(startToken);
+ expression.setEndToken(endToken);
+ setExpressionType(expression);
+ return result;
+ }
+
+ /**
* Evaluates an <a href="http://camel.apache.org/xpath.html";>XPath
* expression</a>
*
Modified:
camel/trunk/camel-core/src/main/java/org/apache/camel/language/tokenizer/TokenizeLanguage.java
URL:
http://svn.apache.org/viewvc/camel/trunk/camel-core/src/main/java/org/apache/camel/language/tokenizer/TokenizeLanguage.java?rev=1194882&r1=1194881&r2=1194882&view=diff
==============================================================================
---
camel/trunk/camel-core/src/main/java/org/apache/camel/language/tokenizer/TokenizeLanguage.java
(original)
+++
camel/trunk/camel-core/src/main/java/org/apache/camel/language/tokenizer/TokenizeLanguage.java
Sat Oct 29 12:37:21 2011
@@ -26,10 +26,19 @@ import org.apache.camel.util.ObjectHelpe
/**
* A language for tokenizer expressions.
+ * <p/>
+ * This tokenizer language can operator in two modes
+ * <ul>
+ * <li>default - using a single tokenizer</li>
+ * <li>pair - using both start and end tokens</li>
+ * </ul>
+ * The default mode supports the <tt>headerName</tt> and <tt>regex</tt>
options.
+ * Where as the pair mode only supports <tt>token</tt> and <tt>endToken</tt>.
*/
public class TokenizeLanguage implements Language, IsSingleton {
private String token;
+ private String endToken;
private String headerName;
private boolean regex;
@@ -56,6 +65,13 @@ public class TokenizeLanguage implements
return language.createExpression(null);
}
+ public static Expression tokenizePair(String startToken, String endToken) {
+ TokenizeLanguage language = new TokenizeLanguage();
+ language.setToken(startToken);
+ language.setEndToken(endToken);
+ return language.createExpression(null);
+ }
+
public Predicate createPredicate(String expression) {
return
ExpressionToPredicateAdapter.toPredicate(createExpression(expression));
}
@@ -65,6 +81,13 @@ public class TokenizeLanguage implements
*/
public Expression createExpression() {
ObjectHelper.notNull(token, "token");
+
+ // if end token is provided then use the tokenize pair expression
+ if (endToken != null) {
+ return ExpressionBuilder.tokenizePairExpression(token, endToken);
+ }
+
+ // use the regular tokenizer
Expression exp = headerName == null ?
ExpressionBuilder.bodyExpression() :
ExpressionBuilder.headerExpression(headerName);
if (regex) {
return ExpressionBuilder.regexTokenizeExpression(exp, token);
@@ -88,6 +111,14 @@ public class TokenizeLanguage implements
this.token = token;
}
+ public String getEndToken() {
+ return endToken;
+ }
+
+ public void setEndToken(String endToken) {
+ this.endToken = endToken;
+ }
+
public String getHeaderName() {
return headerName;
}
Modified:
camel/trunk/camel-core/src/main/java/org/apache/camel/model/language/TokenizerExpression.java
URL:
http://svn.apache.org/viewvc/camel/trunk/camel-core/src/main/java/org/apache/camel/model/language/TokenizerExpression.java?rev=1194882&r1=1194881&r2=1194882&view=diff
==============================================================================
---
camel/trunk/camel-core/src/main/java/org/apache/camel/model/language/TokenizerExpression.java
(original)
+++
camel/trunk/camel-core/src/main/java/org/apache/camel/model/language/TokenizerExpression.java
Sat Oct 29 12:37:21 2011
@@ -26,9 +26,9 @@ import org.apache.camel.Expression;
import org.apache.camel.language.tokenizer.TokenizeLanguage;
/**
- * For expressions and predicates using a body or header tokenizer
+ * For expressions and predicates using a body or header tokenizer.
*
- * @version
+ * @see TokenizeLanguage
*/
@XmlRootElement(name = "tokenize")
@XmlAccessorType(XmlAccessType.FIELD)
@@ -36,6 +36,8 @@ public class TokenizerExpression extends
@XmlAttribute(required = true)
private String token;
@XmlAttribute
+ private String endToken;
+ @XmlAttribute
private String headerName;
@XmlAttribute
private Boolean regex;
@@ -56,6 +58,14 @@ public class TokenizerExpression extends
this.token = token;
}
+ public String getEndToken() {
+ return endToken;
+ }
+
+ public void setEndToken(String endToken) {
+ this.endToken = endToken;
+ }
+
public String getHeaderName() {
return headerName;
}
@@ -76,6 +86,7 @@ public class TokenizerExpression extends
public Expression createExpression(CamelContext camelContext) {
TokenizeLanguage language = new TokenizeLanguage();
language.setToken(token);
+ language.setEndToken(endToken);
language.setHeaderName(headerName);
if (regex != null) {
language.setRegex(regex);
@@ -85,6 +96,10 @@ public class TokenizerExpression extends
@Override
public String toString() {
- return "tokenize{" + (headerName != null ? "header: " + headerName :
"body()") + " using token: " + token + "}";
+ if (endToken != null) {
+ return "tokenize{body() using tokens: " + token + "..." + endToken
+ "}";
+ } else {
+ return "tokenize{" + (headerName != null ? "header: " + headerName
: "body()") + " using token: " + token + "}";
+ }
}
}
\ No newline at end of file
Added:
camel/trunk/camel-core/src/main/java/org/apache/camel/processor/TokenPairExpressionIterator.java
URL:
http://svn.apache.org/viewvc/camel/trunk/camel-core/src/main/java/org/apache/camel/processor/TokenPairExpressionIterator.java?rev=1194882&view=auto
==============================================================================
---
camel/trunk/camel-core/src/main/java/org/apache/camel/processor/TokenPairExpressionIterator.java
(added)
+++
camel/trunk/camel-core/src/main/java/org/apache/camel/processor/TokenPairExpressionIterator.java
Sat Oct 29 12:37:21 2011
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.processor;
+
+import java.io.InputStream;
+import java.util.Iterator;
+import java.util.Scanner;
+
+import org.apache.camel.Exchange;
+import org.apache.camel.InvalidPayloadException;
+import org.apache.camel.support.ExpressionAdapter;
+import org.apache.camel.util.ObjectHelper;
+
+/**
+ * {@link org.apache.camel.Expression} to walk a {@link
org.apache.camel.Message} body
+ * using an {@link Iterator}, which grabs the content between a start and end
token.
+ * <p/>
+ * The message body must be able to convert to {@link InputStream} type which
is used as stream
+ * to access the message body.
+ * <p/>
+ * Can be used to split big XML files
+ */
+public class TokenPairExpressionIterator extends ExpressionAdapter {
+
+ private final String startToken;
+ private final String endToken;
+
+ public TokenPairExpressionIterator(String startToken, String endToken) {
+ this.startToken = startToken;
+ this.endToken = endToken;
+ ObjectHelper.notEmpty(startToken, "startToken");
+ ObjectHelper.notEmpty(endToken, "endToken");
+ }
+
+ @Override
+ public Object evaluate(Exchange exchange) {
+ try {
+ InputStream in =
exchange.getIn().getMandatoryBody(InputStream.class);
+ return new TokenPairIterator(startToken, endToken, in);
+ } catch (InvalidPayloadException e) {
+ exchange.setException(e);
+ return null;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "tokenize[body() using tokens: " + startToken + "..." +
endToken + "]";
+ }
+
+ /**
+ * Iterator to walk the input stream
+ */
+ private static final class TokenPairIterator implements Iterator {
+
+ private final String startToken;
+ private final String endToken;
+ private final Scanner scanner;
+ private Object image;
+
+ private TokenPairIterator(String startToken, String endToken,
InputStream in) {
+ this.startToken = startToken;
+ this.endToken = endToken;
+ // use end token as delimiter
+ this.scanner = new Scanner(in).useDelimiter(endToken);
+ // this iterator will do look ahead as we may have data
+ // after the last end token, which the scanner would find
+ // so we need to be one step ahead of the scanner
+ this.image = hasNext() ? next() : null;
+ }
+
+ @Override
+ public boolean hasNext() {
+ // must look a head
+ boolean answer = scanner.hasNext();
+ if (!answer) {
+ scanner.close();
+ }
+ return answer;
+ }
+
+ @Override
+ public Object next() {
+ Object answer = image;
+ // calculate next
+ image = hasNext() ? getNext() : null;
+
+ if (answer == null) {
+ // first time the image may be null
+ answer = image;
+ }
+ return answer;
+ }
+
+ private Object getNext() {
+ String next = scanner.next();
+
+ // only grab text after the start token
+ if (next != null && next.contains(startToken)) {
+ next = ObjectHelper.after(next, startToken);
+ }
+
+ // include tokens in answer
+ if (next != null) {
+ StringBuilder sb = new StringBuilder();
+ next =
sb.append(startToken).append(next).append(endToken).toString();
+ }
+ return next;
+ }
+
+ @Override
+ public void remove() {
+ // noop
+ }
+ }
+
+}
Modified:
camel/trunk/camel-core/src/test/java/org/apache/camel/component/properties/PropertiesComponentTest.java
URL:
http://svn.apache.org/viewvc/camel/trunk/camel-core/src/test/java/org/apache/camel/component/properties/PropertiesComponentTest.java?rev=1194882&r1=1194881&r2=1194882&view=diff
==============================================================================
---
camel/trunk/camel-core/src/test/java/org/apache/camel/component/properties/PropertiesComponentTest.java
(original)
+++
camel/trunk/camel-core/src/test/java/org/apache/camel/component/properties/PropertiesComponentTest.java
Sat Oct 29 12:37:21 2011
@@ -434,6 +434,37 @@ public class PropertiesComponentTest ext
}
}
+ public void testCache() throws Exception {
+ PropertiesComponent pc = context.getComponent("properties",
PropertiesComponent.class);
+ assertTrue(pc.isCache());
+ assertNotNull(pc);
+
+ for (int i = 0; i < 2000; i++) {
+ String uri = pc.parseUri("{{cool.mock}}:" + i);
+ assertEquals("mock:" + i, uri);
+ }
+ }
+
+ public void testCacheRoute() throws Exception {
+ context.addRoutes(new RouteBuilder() {
+ @Override
+ public void configure() throws Exception {
+ from("direct:start")
+ .setBody(simple("${properties:cool.mock}${body}"))
+ .to("mock:result");
+ }
+ });
+ context.start();
+
+ getMockEndpoint("mock:result").expectedMessageCount(2000);
+
+ for (int i = 0; i < 2000; i++) {
+ template.sendBody("direct:start", i);
+ }
+
+ assertMockEndpointsSatisfied();
+ }
+
@Override
protected CamelContext createCamelContext() throws Exception {
CamelContext context = super.createCamelContext();
Added:
camel/trunk/camel-core/src/test/java/org/apache/camel/language/TokenPairIteratorSplitChoicePerformanceTest.java
URL:
http://svn.apache.org/viewvc/camel/trunk/camel-core/src/test/java/org/apache/camel/language/TokenPairIteratorSplitChoicePerformanceTest.java?rev=1194882&view=auto
==============================================================================
---
camel/trunk/camel-core/src/test/java/org/apache/camel/language/TokenPairIteratorSplitChoicePerformanceTest.java
(added)
+++
camel/trunk/camel-core/src/test/java/org/apache/camel/language/TokenPairIteratorSplitChoicePerformanceTest.java
Sat Oct 29 12:37:21 2011
@@ -0,0 +1,183 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.language;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.camel.ContextTestSupport;
+import org.apache.camel.Exchange;
+import org.apache.camel.Processor;
+import org.apache.camel.builder.NotifyBuilder;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.util.StopWatch;
+import org.apache.camel.util.TimeUtils;
+import org.slf4j.Logger;
+
+/**
+ *
+ */
+public class TokenPairIteratorSplitChoicePerformanceTest extends
ContextTestSupport {
+
+ private int size = 20 * 1000;
+ private final AtomicInteger tiny = new AtomicInteger();
+ private final AtomicInteger small = new AtomicInteger();
+ private final AtomicInteger med = new AtomicInteger();
+ private final AtomicInteger large = new AtomicInteger();
+ private final StopWatch watch = new StopWatch();
+
+ @Override
+ public void setUp() throws Exception {
+ createDataFile(log, size);
+ super.setUp();
+ }
+
+ public void testDummy() {
+ // this is a manual test
+ }
+
+ public void xxxtestTokenPairPerformanceRoute() throws Exception {
+ NotifyBuilder notify = new
NotifyBuilder(context).whenDone(size).create();
+
+ boolean matches = notify.matches(5, TimeUnit.MINUTES);
+ log.info("Processed file with " + size + " elements in: " +
TimeUtils.printDuration(watch.stop()));
+
+ log.info("Processed " + tiny.get() + " tiny messages");
+ log.info("Processed " + small.get() + " small messages");
+ log.info("Processed " + med.get() + " medium messages");
+ log.info("Processed " + large.get() + " large messages");
+
+ assertEquals((size / 10) * 4, tiny.get());
+ assertEquals((size / 10) * 2, small.get());
+ assertEquals((size / 10) * 3, med.get());
+ assertEquals((size / 10) * 1, large.get());
+
+ assertTrue("Should complete route", matches);
+ }
+
+ @Override
+ protected RouteBuilder createRouteBuilder() throws Exception {
+ return new RouteBuilder() {
+ @Override
+ public void configure() throws Exception {
+ from("file:target/data?noop=true")
+ .process(new Processor() {
+ public void process(Exchange exchange) throws
Exception {
+ log.info("Starting to process file");
+ watch.restart();
+ }
+ })
+ .split().tokenizePair("<order>", "</order>").streaming()
+ .choice()
+ .when().xpath("/order/amount < 10")
+ .process(new Processor() {
+ public void process(Exchange exchange)
throws Exception {
+ String xml =
exchange.getIn().getBody(String.class);
+ assertTrue(xml,
xml.contains("<amount>3</amount>"));
+
+ int num = tiny.incrementAndGet();
+ if (num % 100 == 0) {
+ log.info("Processed " + num + "
tiny messages");
+ log.debug(xml);
+ }
+ }
+ })
+ .when().xpath("/order/amount < 50")
+ .process(new Processor() {
+ public void process(Exchange exchange)
throws Exception {
+ String xml =
exchange.getIn().getBody(String.class);
+ assertTrue(xml,
xml.contains("<amount>44</amount>"));
+
+ int num = small.incrementAndGet();
+ if (num % 100 == 0) {
+ log.info("Processed " + num + "
small messages: " + xml);
+ log.debug(xml);
+ }
+ }
+ })
+ .when().xpath("/order/amount < 100")
+ .process(new Processor() {
+ public void process(Exchange exchange)
throws Exception {
+ String xml =
exchange.getIn().getBody(String.class);
+ assertTrue(xml,
xml.contains("<amount>88</amount>"));
+
+ int num = med.incrementAndGet();
+ if (num % 100 == 0) {
+ log.info("Processed " + num + "
medium messages");
+ log.debug(xml);
+ }
+ }
+ })
+ .otherwise()
+ .process(new Processor() {
+ public void process(Exchange exchange)
throws Exception {
+ String xml =
exchange.getIn().getBody(String.class);
+ assertTrue(xml,
xml.contains("<amount>123</amount>"));
+
+ int num = large.incrementAndGet();
+ if (num % 100 == 0) {
+ log.info("Processed " + num + "
large messages");
+ log.debug(xml);
+ }
+ }
+ })
+ .end() // choice
+ .end(); // split
+ }
+ };
+ }
+
+ public static void createDataFile(Logger log, int size) throws Exception {
+ deleteDirectory("target/data");
+ createDirectory("target/data");
+
+ log.info("Creating data file ...");
+
+ File file = new File("target/data/data.xml");
+ FileOutputStream fos = new FileOutputStream(file, true);
+ fos.write("<orders>\n".getBytes());
+
+ for (int i = 0; i < size; i++) {
+ fos.write("<order>\n".getBytes());
+ fos.write((" <id>" + i + "</id>\n").getBytes());
+ int num = i % 10;
+ if (num >= 0 && num <= 3) {
+ fos.write(" <amount>3</amount>\n".getBytes());
+ fos.write(" <customerId>333</customerId>\n".getBytes());
+ } else if (num >= 4 && num <= 5) {
+ fos.write(" <amount>44</amount>\n".getBytes());
+ fos.write(" <customerId>444</customerId>\n".getBytes());
+ } else if (num >= 6 && num <= 8) {
+ fos.write(" <amount>88</amount>\n".getBytes());
+ fos.write(" <customerId>888</customerId>\n".getBytes());
+ } else {
+ fos.write(" <amount>123</amount>\n".getBytes());
+ fos.write(" <customerId>123123</customerId>\n".getBytes());
+ }
+ fos.write(" <description>bla bla bla bla bla bla bla bla bla bla
bla bla bla bla bla bla</description>\n".getBytes());
+ fos.write("</order>\n".getBytes());
+ }
+
+ fos.write("</orders>".getBytes());
+ fos.close();
+
+ log.info("Creating data file done.");
+ }
+
+}
Modified:
camel/trunk/camel-core/src/test/java/org/apache/camel/language/TokenizerTest.java
URL:
http://svn.apache.org/viewvc/camel/trunk/camel-core/src/test/java/org/apache/camel/language/TokenizerTest.java?rev=1194882&r1=1194881&r2=1194882&view=diff
==============================================================================
---
camel/trunk/camel-core/src/test/java/org/apache/camel/language/TokenizerTest.java
(original)
+++
camel/trunk/camel-core/src/test/java/org/apache/camel/language/TokenizerTest.java
Sat Oct 29 12:37:21 2011
@@ -99,4 +99,51 @@ public class TokenizerTest extends Excha
assertEquals(false, lan.isSingleton());
}
+ public void testTokenizePair() throws Exception {
+ Expression exp = TokenizeLanguage.tokenizePair("<person>",
"</person>");
+
+
exchange.getIn().setBody("<persons><person>James</person><person>Claus</person><person>Jonathan</person><person>Hadrian</person></persons>");
+
+ List names = exp.evaluate(exchange, List.class);
+ assertEquals(4, names.size());
+
+ assertEquals("<person>James</person>", names.get(0));
+ assertEquals("<person>Claus</person>", names.get(1));
+ assertEquals("<person>Jonathan</person>", names.get(2));
+ assertEquals("<person>Hadrian</person>", names.get(3));
+ }
+
+ public void testTokenizePairWithNoise() throws Exception {
+ Expression exp = TokenizeLanguage.tokenizePair("<person>",
"</person>");
+
+ exchange.getIn().setBody("<?xml version=\"1.0\"?><!-- bla bla
--><persons>\n<person>James</person>\n<person>Claus</person>\n"
+ + "<!-- more bla bla
--><person>Jonathan</person>\n<person>Hadrian</person>\n</persons> ");
+
+ List names = exp.evaluate(exchange, List.class);
+ assertEquals(4, names.size());
+
+ assertEquals("<person>James</person>", names.get(0));
+ assertEquals("<person>Claus</person>", names.get(1));
+ assertEquals("<person>Jonathan</person>", names.get(2));
+ assertEquals("<person>Hadrian</person>", names.get(3));
+ }
+
+ public void testTokenizePairEmpty() throws Exception {
+ Expression exp = TokenizeLanguage.tokenizePair("<person>",
"</person>");
+
+ exchange.getIn().setBody("<?xml version=\"1.0\"?><!-- bla bla
--><persons></persons> ");
+
+ List names = exp.evaluate(exchange, List.class);
+ assertEquals(0, names.size());
+ }
+
+ public void testTokenizePairNoData() throws Exception {
+ Expression exp = TokenizeLanguage.tokenizePair("<person>",
"</person>");
+
+ exchange.getIn().setBody("");
+
+ List names = exp.evaluate(exchange, List.class);
+ assertNull(names);
+ }
+
}
\ No newline at end of file
Added:
camel/trunk/camel-core/src/test/java/org/apache/camel/language/XPathSplitChoicePerformanceTest.java
URL:
http://svn.apache.org/viewvc/camel/trunk/camel-core/src/test/java/org/apache/camel/language/XPathSplitChoicePerformanceTest.java?rev=1194882&view=auto
==============================================================================
---
camel/trunk/camel-core/src/test/java/org/apache/camel/language/XPathSplitChoicePerformanceTest.java
(added)
+++
camel/trunk/camel-core/src/test/java/org/apache/camel/language/XPathSplitChoicePerformanceTest.java
Sat Oct 29 12:37:21 2011
@@ -0,0 +1,183 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.language;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.camel.ContextTestSupport;
+import org.apache.camel.Exchange;
+import org.apache.camel.Processor;
+import org.apache.camel.builder.NotifyBuilder;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.util.StopWatch;
+import org.apache.camel.util.TimeUtils;
+import org.slf4j.Logger;
+
+/**
+ *
+ */
+public class XPathSplitChoicePerformanceTest extends ContextTestSupport {
+
+ private int size = 20 * 1000;
+ private final AtomicInteger tiny = new AtomicInteger();
+ private final AtomicInteger small = new AtomicInteger();
+ private final AtomicInteger med = new AtomicInteger();
+ private final AtomicInteger large = new AtomicInteger();
+ private final StopWatch watch = new StopWatch();
+
+ @Override
+ public void setUp() throws Exception {
+ createDataFile(log, size);
+ super.setUp();
+ }
+
+ public void testDummy() {
+ // this is a manual test
+ }
+
+ public void xxTestXPatPerformanceRoute() throws Exception {
+ NotifyBuilder notify = new
NotifyBuilder(context).whenDone(size).create();
+
+ boolean matches = notify.matches(60, TimeUnit.SECONDS);
+ log.info("Processed file with " + size + " elements in: " +
TimeUtils.printDuration(watch.stop()));
+
+ log.info("Processed " + tiny.get() + " tiny messages");
+ log.info("Processed " + small.get() + " small messages");
+ log.info("Processed " + med.get() + " medium messages");
+ log.info("Processed " + large.get() + " large messages");
+
+ assertEquals((size / 10) * 4, tiny.get());
+ assertEquals((size / 10) * 2, small.get());
+ assertEquals((size / 10) * 3, med.get());
+ assertEquals((size / 10) * 1, large.get());
+
+ assertTrue("Should complete route", matches);
+ }
+
+ @Override
+ protected RouteBuilder createRouteBuilder() throws Exception {
+ return new RouteBuilder() {
+ @Override
+ public void configure() throws Exception {
+ from("file:target/data?noop=true")
+ .process(new Processor() {
+ public void process(Exchange exchange) throws
Exception {
+ log.info("Starting to process file");
+ watch.restart();
+ }
+ })
+ .split().xpath("/orders/order").streaming()
+ .choice()
+ .when().xpath("/order/amount < 10")
+ .process(new Processor() {
+ public void process(Exchange exchange)
throws Exception {
+ String xml =
exchange.getIn().getBody(String.class);
+ assertTrue(xml,
xml.contains("<amount>3</amount>"));
+
+ int num = tiny.incrementAndGet();
+ if (num % 100 == 0) {
+ log.info("Processed " + num + "
tiny messages");
+ log.debug(xml);
+ }
+ }
+ })
+ .when().xpath("/order/amount < 50")
+ .process(new Processor() {
+ public void process(Exchange exchange)
throws Exception {
+ String xml =
exchange.getIn().getBody(String.class);
+ assertTrue(xml,
xml.contains("<amount>44</amount>"));
+
+ int num = small.incrementAndGet();
+ if (num % 100 == 0) {
+ log.info("Processed " + num + "
small messages");
+ log.debug(xml);
+ }
+ }
+ })
+ .when().xpath("/order/amount < 100")
+ .process(new Processor() {
+ public void process(Exchange exchange)
throws Exception {
+ String xml =
exchange.getIn().getBody(String.class);
+ assertTrue(xml,
xml.contains("<amount>88</amount>"));
+
+ int num = med.incrementAndGet();
+ if (num % 100 == 0) {
+ log.info("Processed " + num + "
medium messages");
+ log.debug(xml);
+ }
+ }
+ })
+ .otherwise()
+ .process(new Processor() {
+ public void process(Exchange exchange)
throws Exception {
+ String xml =
exchange.getIn().getBody(String.class);
+ assertTrue(xml,
xml.contains("<amount>123</amount>"));
+
+ int num = large.incrementAndGet();
+ if (num % 100 == 0) {
+ log.info("Processed " + num + "
large messages");
+ log.debug(xml);
+ }
+ }
+ })
+ .end() // choice
+ .end(); // split
+ }
+ };
+ }
+
+ public static void createDataFile(Logger log, int size) throws Exception {
+ deleteDirectory("target/data");
+ createDirectory("target/data");
+
+ log.info("Creating data file ...");
+
+ File file = new File("target/data/data.xml");
+ FileOutputStream fos = new FileOutputStream(file, true);
+ fos.write("<orders>\n".getBytes());
+
+ for (int i = 0; i < size; i++) {
+ fos.write("<order>\n".getBytes());
+ fos.write((" <id>" + i + "</id>\n").getBytes());
+ int num = i % 10;
+ if (num >= 0 && num <= 3) {
+ fos.write(" <amount>3</amount>\n".getBytes());
+ fos.write(" <customerId>333</customerId>\n".getBytes());
+ } else if (num >= 4 && num <= 5) {
+ fos.write(" <amount>44</amount>\n".getBytes());
+ fos.write(" <customerId>444</customerId>\n".getBytes());
+ } else if (num >= 6 && num <= 8) {
+ fos.write(" <amount>88</amount>\n".getBytes());
+ fos.write(" <customerId>888</customerId>\n".getBytes());
+ } else {
+ fos.write(" <amount>123</amount>\n".getBytes());
+ fos.write(" <customerId>123123</customerId>\n".getBytes());
+ }
+ fos.write(" <description>bla bla bla bla bla bla bla bla bla bla
bla bla bla bla bla bla</description>\n".getBytes());
+ fos.write("</order>\n".getBytes());
+ }
+
+ fos.write("</orders>".getBytes());
+ fos.close();
+
+ log.info("Creating data file done.");
+ }
+
+}
Added:
camel/trunk/components/camel-saxon/src/test/java/org/apache/camel/builder/saxon/XPathSplitChoicePerformanceTest.java
URL:
http://svn.apache.org/viewvc/camel/trunk/components/camel-saxon/src/test/java/org/apache/camel/builder/saxon/XPathSplitChoicePerformanceTest.java?rev=1194882&view=auto
==============================================================================
---
camel/trunk/components/camel-saxon/src/test/java/org/apache/camel/builder/saxon/XPathSplitChoicePerformanceTest.java
(added)
+++
camel/trunk/components/camel-saxon/src/test/java/org/apache/camel/builder/saxon/XPathSplitChoicePerformanceTest.java
Sat Oct 29 12:37:21 2011
@@ -0,0 +1,183 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.builder.saxon;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.camel.Exchange;
+import org.apache.camel.Processor;
+import org.apache.camel.builder.NotifyBuilder;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.test.junit4.CamelTestSupport;
+import org.apache.camel.util.StopWatch;
+import org.apache.camel.util.TimeUtils;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+
+/**
+ *
+ */
+public class XPathSplitChoicePerformanceTest extends CamelTestSupport {
+
+ private int size = 20 * 1000;
+ private final AtomicInteger tiny = new AtomicInteger();
+ private final AtomicInteger small = new AtomicInteger();
+ private final AtomicInteger med = new AtomicInteger();
+ private final AtomicInteger large = new AtomicInteger();
+ private final StopWatch watch = new StopWatch();
+
+ @Override
+ public void setUp() throws Exception {
+ createDataFile(log, size);
+ super.setUp();
+ }
+
+ @Test
+ @Ignore("Manual test")
+ public void testXPathPerformanceRoute() throws Exception {
+ NotifyBuilder notify = new
NotifyBuilder(context).whenDone(size).create();
+
+ boolean matches = notify.matches(60, TimeUnit.SECONDS);
+ log.info("Processed file with " + size + " elements in: " +
TimeUtils.printDuration(watch.stop()));
+
+ log.info("Processed " + tiny.get() + " tiny messages");
+ log.info("Processed " + small.get() + " small messages");
+ log.info("Processed " + med.get() + " medium messages");
+ log.info("Processed " + large.get() + " large messages");
+
+ assertEquals((size / 10) * 4, tiny.get());
+ assertEquals((size / 10) * 2, small.get());
+ assertEquals((size / 10) * 3, med.get());
+ assertEquals((size / 10) * 1, large.get());
+
+ assertTrue("Should complete route", matches);
+ }
+
+ @Override
+ protected RouteBuilder createRouteBuilder() throws Exception {
+ return new RouteBuilder() {
+ @Override
+ public void configure() throws Exception {
+ from("file:target/data?noop=true")
+ .process(new Processor() {
+ public void process(Exchange exchange) throws
Exception {
+ log.info("Starting to process file");
+ watch.restart();
+ }
+ })
+ .split().xpath("/orders/order").streaming()
+ .choice()
+ .when().xpath("/order/amount < 10")
+ .process(new Processor() {
+ public void process(Exchange exchange)
throws Exception {
+ String xml =
exchange.getIn().getBody(String.class);
+ assertTrue(xml,
xml.contains("<amount>3</amount>"));
+
+ int num = tiny.incrementAndGet();
+ if (num % 100 == 0) {
+ log.info("Processed " + num + "
tiny messages");
+ log.debug(xml);
+ }
+ }
+ })
+ .when().xpath("/order/amount < 50")
+ .process(new Processor() {
+ public void process(Exchange exchange)
throws Exception {
+ String xml =
exchange.getIn().getBody(String.class);
+ assertTrue(xml,
xml.contains("<amount>44</amount>"));
+
+ int num = small.incrementAndGet();
+ if (num % 100 == 0) {
+ log.info("Processed " + num + "
small messages");
+ log.debug(xml);
+ }
+ }
+ })
+ .when().xpath("/order/amount < 100")
+ .process(new Processor() {
+ public void process(Exchange exchange)
throws Exception {
+ String xml =
exchange.getIn().getBody(String.class);
+ assertTrue(xml,
xml.contains("<amount>88</amount>"));
+
+ int num = med.incrementAndGet();
+ if (num % 100 == 0) {
+ log.info("Processed " + num + "
medium messages");
+ log.debug(xml);
+ }
+ }
+ })
+ .otherwise()
+ .process(new Processor() {
+ public void process(Exchange exchange)
throws Exception {
+ String xml =
exchange.getIn().getBody(String.class);
+ assertTrue(xml,
xml.contains("<amount>123</amount>"));
+
+ int num = large.incrementAndGet();
+ if (num % 100 == 0) {
+ log.info("Processed " + num + "
large messages");
+ log.debug(xml);
+ }
+ }
+ })
+ .end() // choice
+ .end(); // split
+ }
+ };
+ }
+
+ public static void createDataFile(Logger log, int size) throws Exception {
+ deleteDirectory("target/data");
+ createDirectory("target/data");
+
+ log.info("Creating data file ...");
+
+ File file = new File("target/data/data.xml");
+ FileOutputStream fos = new FileOutputStream(file, true);
+ fos.write("<orders>\n".getBytes());
+
+ for (int i = 0; i < size; i++) {
+ fos.write("<order>\n".getBytes());
+ fos.write((" <id>" + i + "</id>\n").getBytes());
+ int num = i % 10;
+ if (num >= 0 && num <= 3) {
+ fos.write(" <amount>3</amount>\n".getBytes());
+ fos.write(" <customerId>333</customerId>\n".getBytes());
+ } else if (num >= 4 && num <= 5) {
+ fos.write(" <amount>44</amount>\n".getBytes());
+ fos.write(" <customerId>444</customerId>\n".getBytes());
+ } else if (num >= 6 && num <= 8) {
+ fos.write(" <amount>88</amount>\n".getBytes());
+ fos.write(" <customerId>888</customerId>\n".getBytes());
+ } else {
+ fos.write(" <amount>123</amount>\n".getBytes());
+ fos.write(" <customerId>123123</customerId>\n".getBytes());
+ }
+ fos.write(" <description>bla bla bla bla bla bla bla bla bla bla
bla bla bla bla bla bla</description>\n".getBytes());
+ fos.write("</order>\n".getBytes());
+ }
+
+ fos.write("</orders>".getBytes());
+ fos.close();
+
+ log.info("Creating data file done.");
+ }
+
+}
Propchange:
camel/trunk/components/camel-saxon/src/test/java/org/apache/camel/builder/saxon/XPathSplitChoicePerformanceTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange:
camel/trunk/components/camel-saxon/src/test/java/org/apache/camel/builder/saxon/XPathSplitChoicePerformanceTest.java
------------------------------------------------------------------------------
svn:keywords = Rev Date
