This is an automated email from the ASF dual-hosted git repository. jkevan pushed a commit to branch async-save-export-config in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/async-save-export-config by this push: new ec1c0a15c Perf improvements for oneshot exports ec1c0a15c is described below commit ec1c0a15cd5ff24c55f05848141d83ec1b3b8767 Author: Kevan <ke...@jahia.com> AuthorDate: Thu Mar 30 17:04:15 2023 +0200 Perf improvements for oneshot exports --- .../router/services/ProfileExportServiceImpl.java | 36 ++++++++++++++++++---- .../resources/OSGI-INF/blueprint/blueprint.xml | 2 ++ 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileExportServiceImpl.java b/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileExportServiceImpl.java index 8b9471b54..c3197b61e 100644 --- a/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileExportServiceImpl.java +++ b/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileExportServiceImpl.java @@ -18,9 +18,12 @@ package org.apache.unomi.router.services; import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.unomi.api.PartialList; import org.apache.unomi.api.Profile; import org.apache.unomi.api.PropertyType; +import org.apache.unomi.api.conditions.Condition; import org.apache.unomi.api.services.ConfigSharingService; +import org.apache.unomi.api.services.DefinitionsService; import org.apache.unomi.persistence.spi.PersistenceService; import org.apache.unomi.router.api.ExportConfiguration; import org.apache.unomi.router.api.RouterConstants; @@ -40,28 +43,45 @@ public class ProfileExportServiceImpl implements ProfileExportService { private PersistenceService persistenceService; + private DefinitionsService definitionsService; private ConfigSharingService configSharingService; public void setPersistenceService(PersistenceService persistenceService) { this.persistenceService = persistenceService; } + public void setDefinitionsService(DefinitionsService definitionsService) { + this.definitionsService = definitionsService; + } + public void setConfigSharingService(ConfigSharingService configSharingService) { this.configSharingService = configSharingService; } public String extractProfilesBySegment(ExportConfiguration exportConfiguration) { - List<Profile> profileList = persistenceService.query("segments", (String) exportConfiguration.getProperty("segment"), null, Profile.class); + Collection<PropertyType> propertiesDef = persistenceService.query("target", "profiles", null, PropertyType.class); + + Condition segmentCondition = new Condition(); + segmentCondition.setConditionType(definitionsService.getConditionType("profileSegmentCondition")); + segmentCondition.setParameter("segments", Collections.singletonList((String) exportConfiguration.getProperty("segment"))); + segmentCondition.setParameter("matchType", "in"); + StringBuilder csvContent = new StringBuilder(); - for (Profile profile : profileList) { - csvContent.append(convertProfileToCSVLine(profile, exportConfiguration)); - csvContent.append(RouterUtils.getCharFromLineSeparator(exportConfiguration.getLineSeparator())); + PartialList<Profile> profiles = persistenceService.query(segmentCondition, null, Profile.class, 0, 1000, "10m"); + int counter = 0; + while (profiles != null && profiles.getList().size() > 0) { + List<Profile> scrolledProfiles = profiles.getList(); + for (Profile profile : scrolledProfiles) { + csvContent.append(convertProfileToCSVLine(profile, exportConfiguration, propertiesDef)); + csvContent.append(RouterUtils.getCharFromLineSeparator(exportConfiguration.getLineSeparator())); + } + counter += scrolledProfiles.size(); + profiles = persistenceService.continueScrollQuery(Profile.class, profiles.getScrollIdentifier(), profiles.getScrollTimeValidity()); } - logger.debug("Exporting {} extracted profiles.", profileList.size()); Map execution = new HashMap(); execution.put(RouterConstants.KEY_EXECS_DATE, new Date().getTime()); - execution.put(RouterConstants.KEY_EXECS_EXTRACTED, profileList.size()); + execution.put(RouterConstants.KEY_EXECS_EXTRACTED, counter); exportConfiguration = (ExportConfiguration) RouterUtils.addExecutionEntry(exportConfiguration, execution, Integer.parseInt((String) configSharingService.getProperty(RouterConstants.KEY_HISTORY_SIZE))); persistenceService.save(exportConfiguration); @@ -72,6 +92,10 @@ public class ProfileExportServiceImpl implements ProfileExportService { public String convertProfileToCSVLine(Profile profile, ExportConfiguration exportConfiguration) { // TODO: UNOMI-759 querying this everytimes Collection<PropertyType> propertiesDef = persistenceService.query("target", "profiles", null, PropertyType.class); + return convertProfileToCSVLine(profile, exportConfiguration, propertiesDef); + } + + public String convertProfileToCSVLine(Profile profile, ExportConfiguration exportConfiguration, Collection<PropertyType> propertiesDef) { Map<String, String> mapping = (Map<String, String>) exportConfiguration.getProperty("mapping"); String lineToWrite = ""; for (int i = 0; i < mapping.size(); i++) { diff --git a/extensions/router/router-service/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/extensions/router/router-service/src/main/resources/OSGI-INF/blueprint/blueprint.xml index 74e67907a..8b187cad6 100644 --- a/extensions/router/router-service/src/main/resources/OSGI-INF/blueprint/blueprint.xml +++ b/extensions/router/router-service/src/main/resources/OSGI-INF/blueprint/blueprint.xml @@ -22,6 +22,7 @@ <reference id="persistenceService" interface="org.apache.unomi.persistence.spi.PersistenceService"/> <reference id="configSharingService" interface="org.apache.unomi.api.services.ConfigSharingService"/> + <reference id="definitionsService" interface="org.apache.unomi.api.services.DefinitionsService"/> <bean id="importConfigurationServiceImpl" class="org.apache.unomi.router.services.ImportConfigurationServiceImpl"> <property name="persistenceService" ref="persistenceService"/> @@ -59,6 +60,7 @@ <bean id="profileExportServiceImpl" class="org.apache.unomi.router.services.ProfileExportServiceImpl"> <property name="persistenceService" ref="persistenceService"/> <property name="configSharingService" ref="configSharingService" /> + <property name="definitionsService" ref="definitionsService" /> </bean> <service id="profileExportService" ref="profileExportServiceImpl"> <interfaces>