Repository: incubator-unomi Updated Branches: refs/heads/master 061ee51e0 -> d2a480cf5
UNOMI-101 : Add hasHeader, hasDeleteColumn flags in importConfig Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/91e23a29 Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/91e23a29 Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/91e23a29 Branch: refs/heads/master Commit: 91e23a29e8d769cd10deb13f5983cd0a34a161e5 Parents: 061ee51 Author: Abdelkader Midani <amid...@apache.org> Authored: Tue Jul 4 16:47:38 2017 +0200 Committer: Abdelkader Midani <amid...@apache.org> Committed: Tue Jul 4 16:47:38 2017 +0200 ---------------------------------------------------------------------- .../unomi/router/api/ImportConfiguration.java | 24 ++++++++++++++++++++ .../unomi/router/api/RouterConstants.java | 3 ++- .../ImportRouteCompletionProcessor.java | 8 +++++-- .../core/processor/LineSplitProcessor.java | 23 +++++++++++++++---- .../ProfileImportFromSourceRouteBuilder.java | 2 ++ 5 files changed, 53 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/91e23a29/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/ImportConfiguration.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/ImportConfiguration.java b/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/ImportConfiguration.java index b961cfb..a6850ef 100644 --- a/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/ImportConfiguration.java +++ b/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/ImportConfiguration.java @@ -36,6 +36,8 @@ public class ImportConfiguration extends ImportExportConfiguration { private String mergingProperty; private boolean overwriteExistingProfiles = false; private List<String> propertiesToOverwrite; + private boolean hasHeader = false; + private boolean hasDeleteColumn = false; public String getMergingProperty() { @@ -68,6 +70,11 @@ public class ImportConfiguration extends ImportExportConfiguration { this.overwriteExistingProfiles = overwriteExistingProfiles; } + /** + * Retrieves the import configuration propertiesToOverwrite field. + * + * @return propertiesToOverwrite list. + */ public List<String> getPropertiesToOverwrite() { return propertiesToOverwrite; } @@ -76,4 +83,21 @@ public class ImportConfiguration extends ImportExportConfiguration { this.propertiesToOverwrite = propertiesToOverwrite; } + /** + * Retrieves the hasHeader flag. + * + * @return true if the file imported by the current config has a header line. + */ + public boolean isHasHeader() { + return this.hasHeader; + } + + /** + * Retrieves the hasDeleteColumn flag. + * + * @return true if the file imported by the current config has a delete column. + */ + public boolean isHasDeleteColumn() { + return this.hasDeleteColumn; + } } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/91e23a29/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterConstants.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterConstants.java b/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterConstants.java index 6f6d5f5..0fca0e0 100644 --- a/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterConstants.java +++ b/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterConstants.java @@ -48,10 +48,11 @@ public interface RouterConstants { String DEFAULT_FILE_COLUMN_SEPARATOR = ","; String DEFAULT_FILE_LINE_SEPARATOR = "\n"; - String KEY_HISTORY_SIZE = "historySize"; String KEY_CSV_CONTENT = "csvContent"; String KEY_EXECS = "execs"; Object KEY_EXECS_DATE = "date"; Object KEY_EXECS_EXTRACTED = "extractedProfiles"; + + String SYSTEM_SCOPE = "integration"; } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/91e23a29/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/ImportRouteCompletionProcessor.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/ImportRouteCompletionProcessor.java b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/ImportRouteCompletionProcessor.java index 8323879..ac39312 100644 --- a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/ImportRouteCompletionProcessor.java +++ b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/ImportRouteCompletionProcessor.java @@ -60,16 +60,20 @@ public class ImportRouteCompletionProcessor implements Processor { } } + Integer nbTotal = (Integer) exchange.getProperty("CamelSplitSize"); + if(importConfiguration.isHasHeader()) { + nbTotal--; + } + Map execution = new HashMap(); execution.put("date", ((Date) exchange.getProperty("CamelCreatedTimestamp")).getTime()); - execution.put("totalLinesNb", exchange.getProperty("CamelSplitSize")); + execution.put("totalLinesNb", nbTotal); execution.put("successCount", successCount); execution.put("failureCount", failureCount); execution.put("errors", errors); importConfiguration = (ImportConfiguration) RouterUtils.addExecutionEntry(importConfiguration, execution,executionsHistorySize); - importConfiguration.getExecutions().add(execution); //Set running to false, route is complete if (failureCount > 0 && successCount > 0) { importConfiguration.setStatus(RouterConstants.CONFIG_STATUS_COMPLETE_WITH_ERRORS); http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/91e23a29/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java index e047b63..2eadb91 100644 --- a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java +++ b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java @@ -39,10 +39,17 @@ public class LineSplitProcessor implements Processor { private List<String> propertiesToOverwrite; private String mergingProperty; private boolean overwriteExistingProfiles; + private boolean hasHeader; + private boolean hasDeleteColumn; private String columnSeparator; @Override public void process(Exchange exchange) throws Exception { + + if ((Integer) exchange.getProperty("CamelSplitIndex") == 0 && hasHeader) { + exchange.setProperty(Exchange.ROUTE_STOP, Boolean.TRUE); + return; + } //In case of one shot import we check the header and overwrite import config ImportConfiguration importConfigOneShot = (ImportConfiguration) exchange.getIn().getHeader(RouterConstants.HEADER_IMPORT_CONFIG_ONESHOT); String configType = (String) exchange.getIn().getHeader(RouterConstants.HEADER_CONFIG_TYPE); @@ -52,12 +59,15 @@ public class LineSplitProcessor implements Processor { mergingProperty = importConfigOneShot.getMergingProperty(); overwriteExistingProfiles = importConfigOneShot.isOverwriteExistingProfiles(); columnSeparator = importConfigOneShot.getColumnSeparator(); + hasHeader = importConfigOneShot.isHasHeader(); + hasDeleteColumn = importConfigOneShot.isHasDeleteColumn(); } + String[] profileData = ((String) exchange.getIn().getBody()).split(columnSeparator, -1); ProfileToImport profileToImport = new ProfileToImport(); profileToImport.setItemId(UUID.randomUUID().toString()); profileToImport.setItemType("profile"); - profileToImport.setScope("system"); + profileToImport.setScope(RouterConstants.SYSTEM_SCOPE); if (profileData.length > 0 && StringUtils.isNotBlank(profileData[0])) { if (fieldsMapping.size() != (profileData.length - 1)) { throw new BadProfileDataFormatException("The mapping does not match the number of column : line [" + ((Integer) exchange.getProperty("CamelSplitIndex") + 1) + "]", new Throwable("MAPPING_COLUMN_MATCH")); @@ -72,7 +82,8 @@ public class LineSplitProcessor implements Processor { profileToImport.setMergingProperty(mergingProperty); profileToImport.setPropertiesToOverwrite(propertiesToOverwrite); profileToImport.setOverwriteExistingProfiles(overwriteExistingProfiles); - if (StringUtils.isNotBlank(profileData[profileData.length - 1]) && Boolean.parseBoolean(profileData[profileData.length - 1].trim())) { + if (hasDeleteColumn && StringUtils.isNotBlank(profileData[profileData.length - 1]) && + Boolean.parseBoolean(profileData[profileData.length - 1].trim())) { profileToImport.setProfileToDelete(true); } } else { @@ -102,8 +113,12 @@ public class LineSplitProcessor implements Processor { this.overwriteExistingProfiles = overwriteExistingProfiles; } - public String getMergingProperty() { - return this.mergingProperty; + public void setHasHeader(boolean hasHeader) { + this.hasHeader = hasHeader; + } + + public void setHasDeleteColumn(boolean hasDeleteColumn) { + this.hasDeleteColumn = hasDeleteColumn; } /** http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/91e23a29/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportFromSourceRouteBuilder.java ---------------------------------------------------------------------- diff --git a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportFromSourceRouteBuilder.java b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportFromSourceRouteBuilder.java index e3ec8bd..06c8513 100644 --- a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportFromSourceRouteBuilder.java +++ b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportFromSourceRouteBuilder.java @@ -81,6 +81,8 @@ public class ProfileImportFromSourceRouteBuilder extends RouterAbstractRouteBuil lineSplitProcessor.setPropertiesToOverwrite(importConfiguration.getPropertiesToOverwrite()); lineSplitProcessor.setMergingProperty(importConfiguration.getMergingProperty()); lineSplitProcessor.setColumnSeparator(importConfiguration.getColumnSeparator()); + lineSplitProcessor.setHasHeader(importConfiguration.isHasHeader()); + lineSplitProcessor.setHasDeleteColumn(importConfiguration.isHasDeleteColumn()); String endpoint = (String) importConfiguration.getProperties().get("source");
