DMF-1566 : Flat files import - Ability to define column and line separator
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/d95595fd Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/d95595fd Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/d95595fd Branch: refs/heads/feature-DMF-1343 Commit: d95595fdc27dee4af1ca8741431e188c7ff875bc Parents: e78c014 Author: Abdelkader Midani <amid...@apache.org> Authored: Tue Jun 6 17:35:17 2017 +0200 Committer: Abdelkader Midani <amid...@apache.org> Committed: Mon Jun 12 19:24:29 2017 +0200 ---------------------------------------------------------------------- .../unomi/router/api/ImportConfiguration.java | 39 +++++++++++++++++++- .../core/processor/LineSplitProcessor.java | 12 +++++- .../route/ProfileImportOneShotRouteBuilder.java | 2 +- .../ProfileImportSourceToKafkaRouteBuilder.java | 3 +- .../resources/OSGI-INF/blueprint/blueprint.xml | 4 +- .../services/ProfileImportServiceImpl.java | 2 +- 6 files changed, 55 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/d95595fd/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/ImportConfiguration.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/ImportConfiguration.java b/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/ImportConfiguration.java index 770a7b5..29900f9 100644 --- a/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/ImportConfiguration.java +++ b/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/ImportConfiguration.java @@ -17,7 +17,6 @@ package org.apache.unomi.router.api; import org.apache.unomi.api.Item; -import org.apache.unomi.api.MetadataItem; import java.util.HashMap; import java.util.List; @@ -41,6 +40,9 @@ public class ImportConfiguration extends Item { private String mergingProperty; private boolean overwriteExistingProfiles = false; private List<String> propertiesToOverwrite; + + private String columnSeparator = ","; + private String lineSeparator = "\n"; private boolean active = false; /** @@ -182,4 +184,39 @@ public class ImportConfiguration extends Item { this.propertiesToOverwrite = propertiesToOverwrite; } + /** + * gets the column separator. + */ + public String getColumnSeparator() { + return this.columnSeparator; + } + + /** + * Sets the column separator. + * @param columnSeparator property used to specify a line separator. Defaults to ',' + */ + public void setColumnSeparator(String columnSeparator) { + if(this.columnSeparator !=null) { + this.columnSeparator = columnSeparator; + } + } + + /** + * gets the line separator. + */ + public String getLineSeparator() { + return this.lineSeparator; + } + + /** + * Sets the line separator. + * @param lineSeparator property used to specify a line separator. Defaults to '\n' + */ + public void setLineSeparator(String lineSeparator) { + if(lineSeparator != null) { + this.lineSeparator = lineSeparator; + } + } + + } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/d95595fd/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java index 30f79bf..150ef6d 100644 --- a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java +++ b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java @@ -37,6 +37,7 @@ public class LineSplitProcessor implements Processor { private List<String> propertiesToOverwrite; private String mergingProperty; private boolean overwriteExistingProfiles; + private String columnSeparator; @Override public void process(Exchange exchange) throws Exception { @@ -47,8 +48,9 @@ public class LineSplitProcessor implements Processor { propertiesToOverwrite = importConfigOneShot.getPropertiesToOverwrite(); mergingProperty = importConfigOneShot.getMergingProperty(); overwriteExistingProfiles = importConfigOneShot.isOverwriteExistingProfiles(); + columnSeparator = importConfigOneShot.getColumnSeparator(); } - String[] profileData = ((String)exchange.getIn().getBody()).split(","); + String[] profileData = ((String)exchange.getIn().getBody()).split(columnSeparator); ProfileToImport profileToImport = new ProfileToImport(); profileToImport.setItemId(UUID.randomUUID().toString()); profileToImport.setItemType("profile"); @@ -101,4 +103,12 @@ public class LineSplitProcessor implements Processor { this.mergingProperty = mergingProperty; } + /** + * Sets the line separator. + * @param columnSeparator property used to specify a line separator. Defaults to ',' + */ + public void setColumnSeparator(String columnSeparator) { + this.columnSeparator = columnSeparator; + } + } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/d95595fd/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportOneShotRouteBuilder.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportOneShotRouteBuilder.java b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportOneShotRouteBuilder.java index 288d9d1..d095f3e 100644 --- a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportOneShotRouteBuilder.java +++ b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportOneShotRouteBuilder.java @@ -77,7 +77,7 @@ public class ProfileImportOneShotRouteBuilder extends RouteBuilder { .routeId(IMPORT_ONESHOT_ROUTE_ID) .autoStartup(true) .process(importConfigByFileNameProcessor) - .split(bodyAs(String.class).tokenize("\n")) + .split(bodyAs(String.class).tokenize("${in.header.importConfigOneShot.getLineSeparator}")) .process(lineSplitProcessor) .to("log:org.apache.unomi.router?level=INFO") .marshal(jacksonDataFormat) http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/d95595fd/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportSourceToKafkaRouteBuilder.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportSourceToKafkaRouteBuilder.java b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportSourceToKafkaRouteBuilder.java index 382b69d..37ae59e 100644 --- a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportSourceToKafkaRouteBuilder.java +++ b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportSourceToKafkaRouteBuilder.java @@ -78,11 +78,12 @@ public class ProfileImportSourceToKafkaRouteBuilder extends RouteBuilder { lineSplitProcessor.setOverwriteExistingProfiles(importConfiguration.isOverwriteExistingProfiles()); lineSplitProcessor.setPropertiesToOverwrite(importConfiguration.getPropertiesToOverwrite()); lineSplitProcessor.setMergingProperty(importConfiguration.getMergingProperty()); + lineSplitProcessor.setColumnSeparator(importConfiguration.getColumnSeparator()); from((String) importConfiguration.getProperties().get("source")) .routeId(importConfiguration.getItemId())// This allow identification of the route for manual start/stop .autoStartup(importConfiguration.isActive())// Auto-start if the import configuration is set active - .split(bodyAs(String.class).tokenize("\n")) + .split(bodyAs(String.class).tokenize(importConfiguration.getLineSeparator())) .process(lineSplitProcessor) .to("log:org.apache.unomi.router?level=INFO") .marshal(jacksonDataFormat) http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/d95595fd/extensions/router/router-rest/src/main/resources/OSGI-INF/blueprint/blueprint.xml ---------------------------------------------------------------------- diff --git a/extensions/router/router-rest/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/extensions/router/router-rest/src/main/resources/OSGI-INF/blueprint/blueprint.xml index 7e3b5d1..8728a7f 100644 --- a/extensions/router/router-rest/src/main/resources/OSGI-INF/blueprint/blueprint.xml +++ b/extensions/router/router-rest/src/main/resources/OSGI-INF/blueprint/blueprint.xml @@ -33,7 +33,7 @@ <cm:property-placeholder persistent-id="org.apache.unomi.cluster" update-strategy="reload" placeholder-prefix="${cluster."> <cm:default-properties> - <cm:property name="contextserver.port" value="8181"/> + <cm:property name="cluster.contextserver.port" value="8181"/> </cm:default-properties> </cm:property-placeholder> @@ -74,7 +74,7 @@ <bean id="importConfigurationServiceEndPoint" class="org.apache.unomi.router.rest.ImportConfigurationServiceEndPoint"> <property name="importConfigurationService" ref="importConfigurationService"/> <property name="uploadDir" value="${import.oneshot.uploadDir}"/> - <property name="serverPort" value="${contextserver.port}"/> + <property name="serverPort" value="${cluster.contextserver.port}"/> </bean> </blueprint> http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/d95595fd/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileImportServiceImpl.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileImportServiceImpl.java b/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileImportServiceImpl.java index 8097953..6dcc152 100644 --- a/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileImportServiceImpl.java +++ b/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileImportServiceImpl.java @@ -109,7 +109,7 @@ public class ProfileImportServiceImpl implements ProfileImportService, Synchrono BeanUtils.copyProperties(existingProfile, profileToImport); } else { logger.warn("{} occurences found for profile with {} = {}. Profile import is skipped", existingProfiles.size(), - profileToImport.getMergingProperty(), profileToImport.getProperties().get("firstName")); + profileToImport.getMergingProperty(), profileToImport.getProperties().get(profileToImport.getMergingProperty())); } logger.info("-------------------------------------"); return persistenceService.save(existingProfile);
