Repository: incubator-unomi Updated Branches: refs/heads/master 4ee8adb1c -> bc064864d
UNOMI-102 : Use OpenCsv as an RFC4180 parser Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/bc064864 Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/bc064864 Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/bc064864 Branch: refs/heads/master Commit: bc064864dce1b2bdf79ed531b9029a504590e7f5 Parents: 4ee8adb Author: Abdelkader Midani <amid...@apache.org> Authored: Wed Aug 2 04:31:56 2017 +0200 Committer: Abdelkader Midani <amid...@apache.org> Committed: Wed Aug 2 04:31:56 2017 +0200 ---------------------------------------------------------------------- .../org/apache/unomi/router/api/RouterUtils.java | 7 +++++++ extensions/router/router-core/pom.xml | 19 +++++++++++++++++++ .../core/processor/LineSplitProcessor.java | 12 ++++++++++-- .../services/ProfileExportServiceImpl.java | 11 ++++++++--- 4 files changed, 44 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/bc064864/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterUtils.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterUtils.java b/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterUtils.java index b9df06d..4584d76 100644 --- a/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterUtils.java +++ b/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterUtils.java @@ -59,4 +59,11 @@ public class RouterUtils { } return null; } + + public static String removeQuotes(String propValue) { + if(propValue.charAt(0) == '"' && propValue.charAt(propValue.length()-1) == '"') { + return propValue.substring(1, propValue.length()-1); + } + return propValue; + } } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/bc064864/extensions/router/router-core/pom.xml ---------------------------------------------------------------------- diff --git a/extensions/router/router-core/pom.xml b/extensions/router/router-core/pom.xml index 34550d7..96c1c01 100644 --- a/extensions/router/router-core/pom.xml +++ b/extensions/router/router-core/pom.xml @@ -113,6 +113,25 @@ <version>3.5</version> </dependency> <dependency> + <groupId>commons-collections</groupId> + <artifactId>commons-collections</artifactId> + </dependency> + <dependency> + <groupId>commons-beanutils</groupId> + <artifactId>commons-beanutils</artifactId> + </dependency> + <dependency> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + <version>1.1.1</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.opencsv</groupId> + <artifactId>opencsv</artifactId> + <version>3.10</version> + </dependency> + <dependency> <groupId>org.apache.servicemix.bundles</groupId> <artifactId>org.apache.servicemix.bundles.jsch</artifactId> <version>0.1.54_1</version> http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/bc064864/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java index 9e0fb66..dcd8374 100644 --- a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java +++ b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java @@ -16,6 +16,8 @@ */ package org.apache.unomi.router.core.processor; +import com.opencsv.RFC4180Parser; +import com.opencsv.RFC4180ParserBuilder; import org.apache.camel.Exchange; import org.apache.camel.Processor; import org.apache.camel.component.kafka.KafkaConstants; @@ -79,7 +81,11 @@ public class LineSplitProcessor implements Processor { return; } - String[] profileData = ((String) exchange.getIn().getBody()).split(columnSeparator, -1); + RFC4180Parser rfc4180Parser = new RFC4180ParserBuilder() + .withSeparator(columnSeparator.charAt(0)) + .build(); + + String[] profileData = rfc4180Parser.parseLine(((String) exchange.getIn().getBody())); ProfileToImport profileToImport = new ProfileToImport(); profileToImport.setItemId(UUID.randomUUID().toString()); profileToImport.setItemType("profile"); @@ -100,11 +106,13 @@ public class LineSplitProcessor implements Processor { String multivalueArray = profileData[fieldsMapping.get(fieldMappingKey)].trim(); if (StringUtils.isNoneBlank(multiValueDelimiter) && multiValueDelimiter.length() == 2) { multivalueArray = multivalueArray.replaceAll("\\" + multiValueDelimiter.charAt(0), "").replaceAll("\\" + multiValueDelimiter.charAt(1), ""); + multivalueArray = RouterUtils.removeQuotes(multivalueArray); } String[] valuesArray = multivalueArray.split("\\" + multiValueSeparator); properties.put(fieldMappingKey, valuesArray); } else { - properties.put(fieldMappingKey, profileData[fieldsMapping.get(fieldMappingKey)].trim()); + String singleValue = profileData[fieldsMapping.get(fieldMappingKey)].trim(); + properties.put(fieldMappingKey, RouterUtils.removeQuotes(singleValue)); } } else if (propertyType.getValueTypeId().equals("boolean")) { properties.put(fieldMappingKey, new Boolean(profileData[fieldsMapping.get(fieldMappingKey)].trim())); http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/bc064864/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileExportServiceImpl.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileExportServiceImpl.java b/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileExportServiceImpl.java index 832ed46..9e5b792 100644 --- a/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileExportServiceImpl.java +++ b/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileExportServiceImpl.java @@ -68,13 +68,14 @@ public class ProfileExportServiceImpl extends AbstractCustomServiceImpl implemen for (int i = 0; i < mapping.size(); i++) { String propertyName = mapping.get(String.valueOf(i)); PropertyType propType = RouterUtils.getPropertyTypeById(propertiesDef, propertyName); + Object propertyValue = profile.getProperty(propertyName); if (BooleanUtils.isTrue(propType.isMultivalued())) { - List<String> multiValue = (List<String>) profile.getProperty(propertyName); + List<String> multiValue = (List<String>) propertyValue; lineToWrite += StringUtils.isNotBlank(exportConfiguration.getMultiValueDelimiter()) ? exportConfiguration.getMultiValueDelimiter().charAt(0) : ""; int j = 0; for (String entry : multiValue) { - lineToWrite += entry; + lineToWrite += entry.replaceAll("\"", "\"\""); if (j + 1 < multiValue.size()) { lineToWrite += exportConfiguration.getMultiValueSeparator(); } @@ -83,7 +84,11 @@ public class ProfileExportServiceImpl extends AbstractCustomServiceImpl implemen lineToWrite += StringUtils.isNotBlank(exportConfiguration.getMultiValueDelimiter()) ? exportConfiguration.getMultiValueDelimiter().charAt(1) : ""; } else { - lineToWrite += profile.getProperty(propertyName) != null ? profile.getProperty(propertyName) : ""; + propertyValue = propertyValue.toString().replaceAll("\"", "\"\""); + if (StringUtils.contains(propertyValue.toString(), exportConfiguration.getColumnSeparator())) { + propertyValue = "\"" + propertyValue + "\""; + } + lineToWrite += propertyValue.toString() != null ? propertyValue.toString() : ""; } if (i + 1 < mapping.size()) { lineToWrite += exportConfiguration.getColumnSeparator();
