Repository: incubator-unomi Updated Branches: refs/heads/master eed5bec78 -> 1e60e8de8
UNOMI-102 : Validate the data type match the property type Import / Export Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/1e60e8de Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/1e60e8de Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/1e60e8de Branch: refs/heads/master Commit: 1e60e8de8f5aaef9ffcfe3a3f5640569d278d8e9 Parents: eed5bec Author: Abdelkader Midani <amid...@apache.org> Authored: Wed Jul 12 15:48:16 2017 +0200 Committer: Abdelkader Midani <amid...@apache.org> Committed: Wed Jul 12 15:48:16 2017 +0200 ---------------------------------------------------------------------- .../router/api/ImportExportConfiguration.java | 40 ++++++++++ .../unomi/router/api/RouterConstants.java | 5 +- .../apache/unomi/router/api/RouterUtils.java | 14 +++- .../api/services/ProfileExportService.java | 7 +- .../router/core/context/RouterCamelContext.java | 13 +++- .../core/processor/LineBuildProcessor.java | 15 +++- .../core/processor/LineSplitProcessor.java | 80 +++++++++++++++++--- .../ProfileExportProducerRouteBuilder.java | 6 +- .../ProfileImportFromSourceRouteBuilder.java | 4 + .../route/ProfileImportOneShotRouteBuilder.java | 1 + .../core/route/RouterAbstractRouteBuilder.java | 8 ++ .../resources/OSGI-INF/blueprint/blueprint.xml | 2 + .../ExportConfigurationServiceEndPoint.java | 10 ++- .../ImportConfigurationServiceEndPoint.java | 2 +- .../resources/OSGI-INF/blueprint/blueprint.xml | 2 + extensions/router/router-service/pom.xml | 4 + .../services/ProfileExportServiceImpl.java | 33 ++++++-- 17 files changed, 216 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/1e60e8de/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/ImportExportConfiguration.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/ImportExportConfiguration.java b/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/ImportExportConfiguration.java index 82e2f0d..a55d2d1 100644 --- a/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/ImportExportConfiguration.java +++ b/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/ImportExportConfiguration.java @@ -34,6 +34,8 @@ public class ImportExportConfiguration extends Item { private Map<String, Object> properties = new HashMap<>(); private String columnSeparator = ","; private String lineSeparator = "\n"; + private String multiValueSeparator = "|"; + private String multiValueDelimiter = ""; private boolean active; private String status; @@ -183,6 +185,44 @@ public class ImportExportConfiguration extends Item { } /** + * Gets the multi value separator. + * + * @return multiValueSeparator multi value separator + */ + public String getMultiValueSeparator() { + return this.multiValueSeparator; + } + + + /** + * Sets the multi value separator. + * + * @param multiValueSeparator multi value separator + */ + public void setMultiValueSeparator(String multiValueSeparator) { + this.multiValueSeparator = multiValueSeparator; + } + + /** + * Gets the multi value delimiter. + * + * @return multiValueDelimiter multi value delimiter + */ + public String getMultiValueDelimiter() { + return this.multiValueDelimiter; + } + + + /** + * Sets the multi value delimiter. + * + * @param multiValueDelimiter multi value delimiter + */ + public void setMultiValueDelimiter(String multiValueDelimiter) { + this.multiValueDelimiter = multiValueDelimiter; + } + + /** * Retrieves the executions */ public List<Map<String, Object>> getExecutions() { http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/1e60e8de/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterConstants.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterConstants.java b/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterConstants.java index 0fca0e0..3b04703 100644 --- a/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterConstants.java +++ b/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterConstants.java @@ -45,14 +45,15 @@ public interface RouterConstants { String HEADER_IMPORT_CONFIG_ONESHOT = "importConfigOneShot"; String IMPORT_ONESHOT_ROUTE_ID = "ONE_SHOT_ROUTE"; - String DEFAULT_FILE_COLUMN_SEPARATOR = ","; + String IMPORT_ONESHOT_UPLOAD_DIR = "oneshotImportUploadDir"; + String DEFAULT_FILE_COLUMN_SEPARATOR = ","; String DEFAULT_FILE_LINE_SEPARATOR = "\n"; String KEY_HISTORY_SIZE = "historySize"; String KEY_CSV_CONTENT = "csvContent"; String KEY_EXECS = "execs"; Object KEY_EXECS_DATE = "date"; - Object KEY_EXECS_EXTRACTED = "extractedProfiles"; + Object KEY_EXECS_EXTRACTED = "extractedProfiles"; String SYSTEM_SCOPE = "integration"; } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/1e60e8de/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterUtils.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterUtils.java b/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterUtils.java index fc31883..b9df06d 100644 --- a/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterUtils.java +++ b/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterUtils.java @@ -16,6 +16,9 @@ */ package org.apache.unomi.router.api; +import org.apache.unomi.api.PropertyType; + +import java.util.Collection; import java.util.Map; /** @@ -42,9 +45,18 @@ public class RouterUtils { public static char getCharFromLineSeparator(String lineSeparator) { char charLineSep = '\n'; - if("\r".equals(lineSeparator)) { + if ("\r".equals(lineSeparator)) { charLineSep = '\r'; } return charLineSep; } + + public static PropertyType getPropertyTypeById(Collection<PropertyType> propertyTypes, String propertyTypeId) { + for (PropertyType propertyType : propertyTypes) { + if (propertyType.getMetadata().getId().equals(propertyTypeId)) { + return propertyType; + } + } + return null; + } } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/1e60e8de/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/services/ProfileExportService.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/services/ProfileExportService.java b/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/services/ProfileExportService.java index 8f2e51b..38aae4a 100644 --- a/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/services/ProfileExportService.java +++ b/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/services/ProfileExportService.java @@ -17,17 +17,18 @@ package org.apache.unomi.router.api.services; import org.apache.unomi.api.Profile; +import org.apache.unomi.api.PropertyType; import org.apache.unomi.router.api.ExportConfiguration; -import java.util.List; +import java.util.Collection; /** * Created by amidani on 30/06/2017. */ public interface ProfileExportService { - String extractProfilesBySegment(ExportConfiguration exportConfiguration); + String extractProfilesBySegment(ExportConfiguration exportConfiguration, Collection<PropertyType> propertiesDef); - String convertProfileToCSVLine(Profile profile, ExportConfiguration exportConfiguration); + String convertProfileToCSVLine(Profile profile, ExportConfiguration exportConfiguration, Collection<PropertyType> propertiesDef); } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/1e60e8de/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/context/RouterCamelContext.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/context/RouterCamelContext.java b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/context/RouterCamelContext.java index 228dc71..1d66cf7 100644 --- a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/context/RouterCamelContext.java +++ b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/context/RouterCamelContext.java @@ -18,12 +18,11 @@ package org.apache.unomi.router.core.context; import org.apache.camel.CamelContext; import org.apache.camel.Route; -import org.apache.camel.component.file.remote.FtpComponent; import org.apache.camel.component.jackson.JacksonDataFormat; import org.apache.camel.core.osgi.OsgiDefaultCamelContext; -import org.apache.camel.impl.DefaultCamelContext; import org.apache.camel.model.RouteDefinition; import org.apache.unomi.api.services.ConfigSharingService; +import org.apache.unomi.api.services.ProfileService; import org.apache.unomi.persistence.spi.PersistenceService; import org.apache.unomi.router.api.ExportConfiguration; import org.apache.unomi.router.api.ImportConfiguration; @@ -60,6 +59,7 @@ public class RouterCamelContext implements SynchronousBundleListener { private ImportExportConfigurationService<ImportConfiguration> importConfigurationService; private ImportExportConfigurationService<ExportConfiguration> exportConfigurationService; private PersistenceService persistenceService; + private ProfileService profileService; private ProfileExportService profileExportService; private JacksonDataFormat jacksonDataFormat; private String uploadDir; @@ -85,7 +85,7 @@ public class RouterCamelContext implements SynchronousBundleListener { public void initCamelContext() throws Exception { logger.info("Initialize Camel Context..."); - configSharingService.setProperty("oneshotImportUploadDir", uploadDir); + configSharingService.setProperty(RouterConstants.IMPORT_ONESHOT_UPLOAD_DIR, uploadDir); configSharingService.setProperty(RouterConstants.KEY_HISTORY_SIZE, execHistorySize); camelContext = new OsgiDefaultCamelContext(bundleContext); @@ -94,6 +94,7 @@ public class RouterCamelContext implements SynchronousBundleListener { //Source ProfileImportFromSourceRouteBuilder builderReader = new ProfileImportFromSourceRouteBuilder(kafkaProps, configType); + builderReader.setProfileService(profileService); builderReader.setImportConfigurationService(importConfigurationService); builderReader.setJacksonDataFormat(jacksonDataFormat); builderReader.setAllowedEndpoints(allowedEndpoints); @@ -102,6 +103,7 @@ public class RouterCamelContext implements SynchronousBundleListener { //One shot import route ProfileImportOneShotRouteBuilder builderOneShot = new ProfileImportOneShotRouteBuilder(kafkaProps, configType); + builderOneShot.setProfileService(profileService); builderOneShot.setImportConfigByFileNameProcessor(importConfigByFileNameProcessor); builderOneShot.setJacksonDataFormat(jacksonDataFormat); builderOneShot.setUploadDir(uploadDir); @@ -129,6 +131,7 @@ public class RouterCamelContext implements SynchronousBundleListener { //Write to destination ProfileExportProducerRouteBuilder profileExportProducerRouteBuilder = new ProfileExportProducerRouteBuilder(kafkaProps, configType); + profileExportProducerRouteBuilder.setProfileService(profileService); profileExportProducerRouteBuilder.setProfileExportService(profileExportService); profileExportProducerRouteBuilder.setExportRouteCompletionProcessor(exportRouteCompletionProcessor); profileExportProducerRouteBuilder.setAllowedEndpoints(allowedEndpoints); @@ -241,6 +244,10 @@ public class RouterCamelContext implements SynchronousBundleListener { this.profileExportService = profileExportService; } + public void setProfileService(ProfileService profileService) { + this.profileService = profileService; + } + public void setJacksonDataFormat(JacksonDataFormat jacksonDataFormat) { this.jacksonDataFormat = jacksonDataFormat; } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/1e60e8de/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineBuildProcessor.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineBuildProcessor.java b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineBuildProcessor.java index b640405..1da4c13 100644 --- a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineBuildProcessor.java +++ b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineBuildProcessor.java @@ -19,11 +19,14 @@ package org.apache.unomi.router.core.processor; import org.apache.camel.Exchange; import org.apache.camel.Processor; import org.apache.unomi.api.Profile; +import org.apache.unomi.api.PropertyType; import org.apache.unomi.router.api.ExportConfiguration; import org.apache.unomi.router.api.services.ProfileExportService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collection; + /** * Created by amidani on 28/06/2017. */ @@ -32,6 +35,7 @@ public class LineBuildProcessor implements Processor { private static final Logger logger = LoggerFactory.getLogger(LineBuildProcessor.class); private ProfileExportService profileExportService; + private Collection<PropertyType> propertiesDef; public LineBuildProcessor(ProfileExportService profileExportService) { this.profileExportService = profileExportService; @@ -42,9 +46,18 @@ public class LineBuildProcessor implements Processor { ExportConfiguration exportConfiguration = (ExportConfiguration) exchange.getIn().getHeader("exportConfig"); Profile profile = exchange.getIn().getBody(Profile.class); - String lineToWrite = profileExportService.convertProfileToCSVLine(profile, exportConfiguration); + String lineToWrite = profileExportService.convertProfileToCSVLine(profile, exportConfiguration, propertiesDef); exchange.getIn().setBody(lineToWrite, String.class); } + /** + * Sets the Property definitions list. + * + * @param propertiesDef Property definitions list + */ + public void setPropertiesDef(Collection<PropertyType> propertiesDef) { + this.propertiesDef = propertiesDef; + } + } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/1e60e8de/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java index 33332bb..df203e3 100644 --- a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java +++ b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java @@ -19,22 +19,26 @@ package org.apache.unomi.router.core.processor; import org.apache.camel.Exchange; import org.apache.camel.Processor; import org.apache.camel.component.kafka.KafkaConstants; +import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.unomi.api.PropertyType; import org.apache.unomi.router.api.ImportConfiguration; import org.apache.unomi.router.api.ProfileToImport; import org.apache.unomi.router.api.RouterConstants; +import org.apache.unomi.router.api.RouterUtils; import org.apache.unomi.router.core.exception.BadProfileDataFormatException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; +import java.util.*; /** * Created by amidani on 29/12/2016. */ public class LineSplitProcessor implements Processor { + private static final Logger logger = LoggerFactory.getLogger(LineSplitProcessor.class.getName()); + private Map<String, Integer> fieldsMapping; private List<String> propertiesToOverwrite; private String mergingProperty; @@ -43,13 +47,14 @@ public class LineSplitProcessor implements Processor { private boolean hasDeleteColumn; private String columnSeparator; + private String multiValueSeparator; + private String multiValueDelimiter; + + private Collection<PropertyType> propertiesDef; + @Override public void process(Exchange exchange) throws Exception { - if ((Integer) exchange.getProperty("CamelSplitIndex") == 0 && hasHeader) { - exchange.setProperty(Exchange.ROUTE_STOP, Boolean.TRUE); - return; - } //In case of one shot import we check the header and overwrite import config ImportConfiguration importConfigOneShot = (ImportConfiguration) exchange.getIn().getHeader(RouterConstants.HEADER_IMPORT_CONFIG_ONESHOT); String configType = (String) exchange.getIn().getHeader(RouterConstants.HEADER_CONFIG_TYPE); @@ -61,6 +66,13 @@ public class LineSplitProcessor implements Processor { columnSeparator = importConfigOneShot.getColumnSeparator(); hasHeader = importConfigOneShot.isHasHeader(); hasDeleteColumn = importConfigOneShot.isHasDeleteColumn(); + multiValueSeparator = importConfigOneShot.getMultiValueSeparator(); + multiValueDelimiter = importConfigOneShot.getMultiValueDelimiter(); + } + + if ((Integer) exchange.getProperty("CamelSplitIndex") == 0 && hasHeader) { + exchange.setProperty(Exchange.ROUTE_STOP, Boolean.TRUE); + return; } String[] profileData = ((String) exchange.getIn().getBody()).split(columnSeparator, -1); @@ -68,14 +80,37 @@ public class LineSplitProcessor implements Processor { profileToImport.setItemId(UUID.randomUUID().toString()); profileToImport.setItemType("profile"); profileToImport.setScope(RouterConstants.SYSTEM_SCOPE); + if (profileData.length > 0 && StringUtils.isNotBlank(profileData[0])) { if (hasDeleteColumn && (fieldsMapping.size() != (profileData.length - 1))) { throw new BadProfileDataFormatException("The mapping does not match the number of column : line [" + ((Integer) exchange.getProperty("CamelSplitIndex") + 1) + "]", new Throwable("MAPPING_COLUMN_MATCH")); } Map<String, Object> properties = new HashMap<>(); for (String fieldMappingKey : fieldsMapping.keySet()) { + PropertyType propertyType = RouterUtils.getPropertyTypeById(propertiesDef, fieldMappingKey); + if (profileData.length > fieldsMapping.get(fieldMappingKey)) { - properties.put(fieldMappingKey, profileData[fieldsMapping.get(fieldMappingKey)].trim()); + try { + if (propertyType.getValueTypeId().equals("string") || propertyType.getValueTypeId().equals("email")) { + if (BooleanUtils.isTrue(propertyType.isMultivalued())) { + String multivalueArray = profileData[fieldsMapping.get(fieldMappingKey)].trim(); + if(StringUtils.isNoneBlank(multiValueDelimiter) && multiValueDelimiter.length() == 2) { + multivalueArray = multivalueArray.replaceAll("\\"+multiValueDelimiter.charAt(0),"").replaceAll("\\"+multiValueDelimiter.charAt(1), ""); + } + String[] valuesArray = multivalueArray.split("\\"+multiValueSeparator); + properties.put(fieldMappingKey, valuesArray); + } else { + properties.put(fieldMappingKey, profileData[fieldsMapping.get(fieldMappingKey)].trim()); + } + } else if (propertyType.getValueTypeId().equals("boolean")) { + properties.put(fieldMappingKey, new Boolean(profileData[fieldsMapping.get(fieldMappingKey)].trim())); + } else if (propertyType.getValueTypeId().equals("integer")) { + properties.put(fieldMappingKey, new Integer(profileData[fieldsMapping.get(fieldMappingKey)].trim())); + } + } catch (Exception e) { + throw new BadProfileDataFormatException("Unable to convert '" + profileData[fieldsMapping.get(fieldMappingKey)].trim() + "' to " + propertyType.getValueTypeId(), new Throwable("DATA_TYPE")); + } + } } profileToImport.setProperties(properties); @@ -139,4 +174,31 @@ public class LineSplitProcessor implements Processor { this.columnSeparator = columnSeparator; } + /** + * Sets the multi value separator. + * + * @param multiValueSeparator multi value separator + */ + public void setMultiValueSeparator(String multiValueSeparator) { + this.multiValueSeparator = multiValueSeparator; + } + + /** + * Sets the multi value delimiter. + * + * @param multiValueDelimiter multi value delimiter + */ + public void setMultiValueDelimiter(String multiValueDelimiter) { + this.multiValueDelimiter = multiValueDelimiter; + } + + /** + * Sets the Property definitions list. + * + * @param propertiesDef Property definitions list + */ + public void setPropertiesDef(Collection<PropertyType> propertiesDef) { + this.propertiesDef = propertiesDef; + } + } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/1e60e8de/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileExportProducerRouteBuilder.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileExportProducerRouteBuilder.java b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileExportProducerRouteBuilder.java index bfcd8ae..50d758e 100644 --- a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileExportProducerRouteBuilder.java +++ b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileExportProducerRouteBuilder.java @@ -59,8 +59,12 @@ public class ProfileExportProducerRouteBuilder extends RouterAbstractRouteBuilde rtDef = from((String) getEndpointURI(RouterConstants.DIRECTION_TO, RouterConstants.DIRECT_EXPORT_DEPOSIT_BUFFER)); } + LineBuildProcessor processor = new LineBuildProcessor(profileExportService); + processor.setPropertiesDef(profileService.getAllPropertyTypes("profiles")); + + rtDef.unmarshal(jacksonDataFormat) - .process(new LineBuildProcessor(profileExportService)) + .process(processor) .aggregate(constant(true), new StringLinesAggregationStrategy()) .completionPredicate(exchangeProperty("CamelSplitSize").isEqualTo(exchangeProperty("CamelAggregatedSize"))) .eagerCheckCompletion() http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/1e60e8de/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportFromSourceRouteBuilder.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportFromSourceRouteBuilder.java b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportFromSourceRouteBuilder.java index 06c8513..5e51c6d 100644 --- a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportFromSourceRouteBuilder.java +++ b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportFromSourceRouteBuilder.java @@ -83,6 +83,10 @@ public class ProfileImportFromSourceRouteBuilder extends RouterAbstractRouteBuil lineSplitProcessor.setColumnSeparator(importConfiguration.getColumnSeparator()); lineSplitProcessor.setHasHeader(importConfiguration.isHasHeader()); lineSplitProcessor.setHasDeleteColumn(importConfiguration.isHasDeleteColumn()); + lineSplitProcessor.setMultiValueDelimiter(importConfiguration.getMultiValueDelimiter()); + lineSplitProcessor.setMultiValueSeparator(importConfiguration.getMultiValueSeparator()); + + lineSplitProcessor.setPropertiesDef(profileService.getAllPropertyTypes("profiles")); String endpoint = (String) importConfiguration.getProperties().get("source"); http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/1e60e8de/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportOneShotRouteBuilder.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportOneShotRouteBuilder.java b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportOneShotRouteBuilder.java index 0913876..0b60671 100644 --- a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportOneShotRouteBuilder.java +++ b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportOneShotRouteBuilder.java @@ -59,6 +59,7 @@ public class ProfileImportOneShotRouteBuilder extends RouterAbstractRouteBuilder } LineSplitProcessor lineSplitProcessor = new LineSplitProcessor(); + lineSplitProcessor.setPropertiesDef(profileService.getAllPropertyTypes("profiles")); ProcessorDefinition prDef = from("file://" + uploadDir + "?include=.*.csv&consumer.delay=1m") .routeId(RouterConstants.IMPORT_ONESHOT_ROUTE_ID) http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/1e60e8de/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/RouterAbstractRouteBuilder.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/RouterAbstractRouteBuilder.java b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/RouterAbstractRouteBuilder.java index 5db9917..ad06a00 100644 --- a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/RouterAbstractRouteBuilder.java +++ b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/RouterAbstractRouteBuilder.java @@ -22,6 +22,7 @@ import org.apache.camel.component.kafka.KafkaComponent; import org.apache.camel.component.kafka.KafkaConfiguration; import org.apache.camel.component.kafka.KafkaEndpoint; import org.apache.commons.lang3.StringUtils; +import org.apache.unomi.api.services.ProfileService; import org.apache.unomi.router.api.RouterConstants; import java.util.Map; @@ -45,6 +46,8 @@ public abstract class RouterAbstractRouteBuilder extends RouteBuilder { protected String configType; protected String allowedEndpoints; + protected ProfileService profileService; + public RouterAbstractRouteBuilder(Map<String, String> kafkaProps, String configType) { this.kafkaHost = kafkaProps.get("kafkaHost"); this.kafkaPort = kafkaProps.get("kafkaPort"); @@ -95,4 +98,9 @@ public abstract class RouterAbstractRouteBuilder extends RouteBuilder { public void setAllowedEndpoints(String allowedEndpoints) { this.allowedEndpoints = allowedEndpoints; } + + public void setProfileService(ProfileService profileService) { + this.profileService = profileService; + } + } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/1e60e8de/extensions/router/router-core/src/main/resources/OSGI-INF/blueprint/blueprint.xml ---------------------------------------------------------------------- diff --git a/extensions/router/router-core/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/extensions/router/router-core/src/main/resources/OSGI-INF/blueprint/blueprint.xml index 29df601..6b19197 100644 --- a/extensions/router/router-core/src/main/resources/OSGI-INF/blueprint/blueprint.xml +++ b/extensions/router/router-core/src/main/resources/OSGI-INF/blueprint/blueprint.xml @@ -117,6 +117,7 @@ <property name="importConfigurationService" ref="importConfigurationService"/> <property name="persistenceService" ref="persistenceService"/> <property name="profileExportService" ref="profileExportService"/> + <property name="profileService" ref="profileService"/> </bean> <camel:camelContext id="httpEndpoint" xmlns="http://camel.apache.org/schema/blueprint";> @@ -135,6 +136,7 @@ <reference id="httpService" interface="org.osgi.service.http.HttpService"/> <reference id="profileImportService" interface="org.apache.unomi.router.api.services.ProfileImportService"/> <reference id="profileExportService" interface="org.apache.unomi.router.api.services.ProfileExportService"/> + <reference id="profileService" interface="org.apache.unomi.api.services.ProfileService"/> <reference id="persistenceService" interface="org.apache.unomi.persistence.spi.PersistenceService"/> <reference id="importConfigurationService" interface="org.apache.unomi.router.api.services.ImportExportConfigurationService" filter="(configDiscriminator=IMPORT)"/> <reference id="exportConfigurationService" interface="org.apache.unomi.router.api.services.ImportExportConfigurationService" filter="(configDiscriminator=EXPORT)"/> http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/1e60e8de/extensions/router/router-rest/src/main/java/org/apache/unomi/router/rest/ExportConfigurationServiceEndPoint.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-rest/src/main/java/org/apache/unomi/router/rest/ExportConfigurationServiceEndPoint.java b/extensions/router/router-rest/src/main/java/org/apache/unomi/router/rest/ExportConfigurationServiceEndPoint.java index 9e52c05..201b341 100644 --- a/extensions/router/router-rest/src/main/java/org/apache/unomi/router/rest/ExportConfigurationServiceEndPoint.java +++ b/extensions/router/router-rest/src/main/java/org/apache/unomi/router/rest/ExportConfigurationServiceEndPoint.java @@ -23,6 +23,7 @@ import org.apache.http.client.methods.HttpPut; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; +import org.apache.unomi.api.services.ProfileService; import org.apache.unomi.router.api.ExportConfiguration; import org.apache.unomi.router.api.services.ImportExportConfigurationService; import org.apache.unomi.router.api.services.ProfileExportService; @@ -53,6 +54,7 @@ public class ExportConfigurationServiceEndPoint extends AbstractConfigurationSer private static final Logger logger = LoggerFactory.getLogger(ExportConfigurationServiceEndPoint.class.getName()); private ProfileExportService profileExportService; + private ProfileService profileService; public ExportConfigurationServiceEndPoint() { logger.info("Initializing export configuration service endpoint..."); @@ -68,6 +70,11 @@ public class ExportConfigurationServiceEndPoint extends AbstractConfigurationSer this.profileExportService = profileExportService; } + @WebMethod(exclude = true) + public void setProfileService(ProfileService profileService) { + this.profileService = profileService; + } + /** * Save the given export configuration. * @@ -111,7 +118,8 @@ public class ExportConfigurationServiceEndPoint extends AbstractConfigurationSer @Consumes(MediaType.APPLICATION_JSON) @Produces("text/csv") public Response processOneshotImportConfigurationCSV(ExportConfiguration exportConfiguration) { - String csvContent = profileExportService.extractProfilesBySegment(exportConfiguration); + String csvContent = profileExportService.extractProfilesBySegment(exportConfiguration, + profileService.getAllPropertyTypes("profiles")); Response.ResponseBuilder response = Response.ok(csvContent); response.header("Content-Disposition", "attachment; filename=Profiles_export_" + new SimpleDateFormat("yyyy-MM-dd-HH-mm").format(new Date()) + ".csv"); http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/1e60e8de/extensions/router/router-rest/src/main/java/org/apache/unomi/router/rest/ImportConfigurationServiceEndPoint.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-rest/src/main/java/org/apache/unomi/router/rest/ImportConfigurationServiceEndPoint.java b/extensions/router/router-rest/src/main/java/org/apache/unomi/router/rest/ImportConfigurationServiceEndPoint.java index 0f8c633..1d13e7f 100644 --- a/extensions/router/router-rest/src/main/java/org/apache/unomi/router/rest/ImportConfigurationServiceEndPoint.java +++ b/extensions/router/router-rest/src/main/java/org/apache/unomi/router/rest/ImportConfigurationServiceEndPoint.java @@ -109,7 +109,7 @@ public class ImportConfigurationServiceEndPoint extends AbstractConfigurationSer @Produces(MediaType.APPLICATION_JSON) public Response processOneshotImportConfigurationCSV(@Multipart(value = "importConfigId") String importConfigId, @Multipart(value = "file") Attachment file) { try { - java.nio.file.Path path = Paths.get(configSharingService.getProperty("oneshotImportUploadDir") + importConfigId + ".csv"); + java.nio.file.Path path = Paths.get(configSharingService.getProperty(RouterConstants.IMPORT_ONESHOT_UPLOAD_DIR) + importConfigId + ".csv"); Files.deleteIfExists(path); InputStream in = file.getObject(InputStream.class); http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/1e60e8de/extensions/router/router-rest/src/main/resources/OSGI-INF/blueprint/blueprint.xml ---------------------------------------------------------------------- diff --git a/extensions/router/router-rest/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/extensions/router/router-rest/src/main/resources/OSGI-INF/blueprint/blueprint.xml index ebf1daf..e1d745d 100644 --- a/extensions/router/router-rest/src/main/resources/OSGI-INF/blueprint/blueprint.xml +++ b/extensions/router/router-rest/src/main/resources/OSGI-INF/blueprint/blueprint.xml @@ -72,6 +72,7 @@ <reference id="configSharingService" interface="org.apache.unomi.api.services.ConfigSharingService"/> <reference id="profileExportService" interface="org.apache.unomi.router.api.services.ProfileExportService"/> + <reference id="profileService" interface="org.apache.unomi.api.services.ProfileService"/> <bean id="importConfigurationServiceEndPoint" class="org.apache.unomi.router.rest.ImportConfigurationServiceEndPoint"> <property name="importConfigurationService" ref="importConfigurationService"/> @@ -82,6 +83,7 @@ <property name="exportConfigurationService" ref="exportConfigurationService"/> <property name="configSharingService" ref="configSharingService"/> <property name="profileExportService" ref="profileExportService"/> + <property name="profileService" ref="profileService"/> </bean> </blueprint> http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/1e60e8de/extensions/router/router-service/pom.xml ---------------------------------------------------------------------- diff --git a/extensions/router/router-service/pom.xml b/extensions/router/router-service/pom.xml index 4ab17ce..ed96bb1 100644 --- a/extensions/router/router-service/pom.xml +++ b/extensions/router/router-service/pom.xml @@ -95,6 +95,10 @@ <artifactId>javax.servlet-api</artifactId> <scope>provided</scope> </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-lang3</artifactId> + </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/1e60e8de/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileExportServiceImpl.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileExportServiceImpl.java b/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileExportServiceImpl.java index efed8c4..832ed46 100644 --- a/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileExportServiceImpl.java +++ b/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileExportServiceImpl.java @@ -16,7 +16,10 @@ */ package org.apache.unomi.router.services; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.unomi.api.Profile; +import org.apache.unomi.api.PropertyType; import org.apache.unomi.api.services.ConfigSharingService; import org.apache.unomi.router.api.ExportConfiguration; import org.apache.unomi.router.api.RouterConstants; @@ -25,10 +28,7 @@ import org.apache.unomi.router.api.services.ProfileExportService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; /** * Created by amidani on 30/06/2017. @@ -39,11 +39,11 @@ public class ProfileExportServiceImpl extends AbstractCustomServiceImpl implemen private ConfigSharingService configSharingService; - public String extractProfilesBySegment(ExportConfiguration exportConfiguration) { + public String extractProfilesBySegment(ExportConfiguration exportConfiguration, Collection<PropertyType> propertiesDef) { List<Profile> profileList = persistenceService.query("segments", (String) exportConfiguration.getProperty("segment"), null, Profile.class); StringBuilder csvContent = new StringBuilder(); for (Profile profile : profileList) { - csvContent.append(convertProfileToCSVLine(profile, exportConfiguration)); + csvContent.append(convertProfileToCSVLine(profile, exportConfiguration, propertiesDef)); csvContent.append(RouterUtils.getCharFromLineSeparator(exportConfiguration.getLineSeparator())); } logger.debug("Exporting {} extracted profiles.", profileList.size()); @@ -62,12 +62,29 @@ public class ProfileExportServiceImpl extends AbstractCustomServiceImpl implemen return csvContent.toString(); } - public String convertProfileToCSVLine(Profile profile, ExportConfiguration exportConfiguration) { + public String convertProfileToCSVLine(Profile profile, ExportConfiguration exportConfiguration, Collection<PropertyType> propertiesDef) { Map<String, String> mapping = (Map<String, String>) exportConfiguration.getProperty("mapping"); String lineToWrite = ""; for (int i = 0; i < mapping.size(); i++) { String propertyName = mapping.get(String.valueOf(i)); - lineToWrite += profile.getProperty(propertyName) != null ? profile.getProperty(propertyName) : ""; + PropertyType propType = RouterUtils.getPropertyTypeById(propertiesDef, propertyName); + if (BooleanUtils.isTrue(propType.isMultivalued())) { + List<String> multiValue = (List<String>) profile.getProperty(propertyName); + + lineToWrite += StringUtils.isNotBlank(exportConfiguration.getMultiValueDelimiter()) ? exportConfiguration.getMultiValueDelimiter().charAt(0) : ""; + int j = 0; + for (String entry : multiValue) { + lineToWrite += entry; + if (j + 1 < multiValue.size()) { + lineToWrite += exportConfiguration.getMultiValueSeparator(); + } + j++; + } + lineToWrite += StringUtils.isNotBlank(exportConfiguration.getMultiValueDelimiter()) ? exportConfiguration.getMultiValueDelimiter().charAt(1) : ""; + + } else { + lineToWrite += profile.getProperty(propertyName) != null ? profile.getProperty(propertyName) : ""; + } if (i + 1 < mapping.size()) { lineToWrite += exportConfiguration.getColumnSeparator(); }
