This is an automated email from the ASF dual-hosted git repository. jkevan pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/master by this push: new 9d1fc3222 UNOMI-736: merge all item indices into same index (#581) 9d1fc3222 is described below commit 9d1fc322279dc424c07a7c2a4dbd95e8098e94de Author: kevan Jahanshahi <jke...@apache.org> AuthorDate: Wed Mar 1 17:45:53 2023 +0100 UNOMI-736: merge all item indices into same index (#581) * DMF-5757: merge all item indices into same index for reducing number of indices * fix bad merge * small fix * UNOMI-736: POC cleanup * UNOMI-736: Fix old migration on 2.0.0 due to mappings removed from core persistence * UNOMI-736: More fixes regarding groovy actions * UNOMI-736: More fixes regarding groovy actions * UNOMI-736: provide base migration for indices reduction * UNOMI-736: provide base migration for indices reduction * UNOMI-736: More fixes regarding groovy actions * UNOMI-736: More fixes regarding groovy actions * UNOMI-736: Fix IT * UNOMI-736: Fix IT * UNOMI-736: Fix IT * UNOMI-736: Add logs to debug Github runs ... Something looks wrong * UNOMI-736: Cleanup annoying blueprint stuff * UNOMI-736: feedback --- .../karaf-kar/src/main/feature/feature.xml | 8 +- extensions/groovy-actions/services/pom.xml | 1 + .../groovy/actions/GroovyActionDispatcher.java | 6 +- .../actions/listener/GroovyActionListener.java | 30 ++--- .../services/impl/GroovyActionsServiceImpl.java | 98 +++++++------- .../META-INF/cxs/mappings/groovyAction.json | 28 ---- .../resources/OSGI-INF/blueprint/blueprint.xml | 69 ---------- .../unomi/schema/impl/SchemaServiceImpl.java | 9 -- itests/pom.xml | 6 + .../unomi/itests/GroovyActionsServiceIT.java | 13 +- .../unomi/itests/migration/Migrate16xTo220IT.java | 23 ++-- kar/pom.xml | 10 -- .../ElasticSearchPersistenceServiceImpl.java | 78 ++++++++++-- .../META-INF/cxs/mappings/actionType.json | 20 --- .../META-INF/cxs/mappings/campaignevent.json | 44 ------- .../META-INF/cxs/mappings/exportConfig.json | 20 --- .../META-INF/cxs/mappings/importConfig.json | 20 --- .../META-INF/cxs/mappings/jsonschema.json | 25 ---- .../resources/META-INF/cxs/mappings/persona.json | 38 ------ .../META-INF/cxs/mappings/personaSession.json | 41 ------ .../META-INF/cxs/mappings/propertyType.json | 61 --------- .../resources/META-INF/cxs/mappings/rulestats.json | 20 --- .../META-INF/cxs/mappings/systemItems.json | 141 +++++++++++++++++++++ .../resources/META-INF/cxs/mappings/topic.json | 20 --- .../services/impl/rules/RulesServiceImpl.java | 24 +++- .../shell/migration/utils/MigrationUtils.java | 13 +- .../migrate-2.0.0-05-globalReindex.groovy | 2 +- .../cxs/migration/migrate-2.0.0-20-scopes.groovy | 2 +- ...-2.2.0-00-rolloverAndMigrateEventSession.groovy | 4 +- .../migrate-2.2.0-05-indicesReduction.groovy | 78 ++++++++++++ .../requestBody/2.0.0}/mappings/campaign.json | 0 .../requestBody/2.0.0}/mappings/conditionType.json | 0 .../requestBody/2.0.0}/mappings/goal.json | 0 .../requestBody/2.0.0}/mappings/patch.json | 0 .../requestBody/2.0.0}/mappings/rule.json | 0 .../requestBody/2.0.0}/mappings/scope.json | 0 .../requestBody/2.0.0}/mappings/scoring.json | 0 .../requestBody/2.0.0}/mappings/segment.json | 0 .../requestBody/2.2.0/base_reindex_request.json | 2 +- .../requestBody/2.2.0/suffix_ids.painless | 19 +++ 40 files changed, 434 insertions(+), 539 deletions(-) diff --git a/extensions/groovy-actions/karaf-kar/src/main/feature/feature.xml b/extensions/groovy-actions/karaf-kar/src/main/feature/feature.xml index a4af565e2..649b74097 100644 --- a/extensions/groovy-actions/karaf-kar/src/main/feature/feature.xml +++ b/extensions/groovy-actions/karaf-kar/src/main/feature/feature.xml @@ -18,15 +18,15 @@ <features xmlns="http://karaf.apache.org/xmlns/features/v1.6.0" name="unomi-groovy-actions"> <feature name="unomi-groovy-actions" description="${project.name}" version="${project.version}"> <details>${project.description}</details> - <feature prerequisite="true" dependency="false">wrap</feature> - <feature dependency="true">unomi-kar</feature> + <feature>wrap</feature> + <feature>unomi-kar</feature> <bundle start-level="85">mvn:org.codehaus.groovy/groovy/${groovy.version}</bundle> <bundle start-level="85">mvn:org.codehaus.groovy/groovy-xml/${groovy.version}</bundle> <bundle start-level="85">mvn:org.codehaus.groovy/groovy-json/${groovy.version}</bundle> - <bundle start-level="85" start="false">mvn:org.apache.unomi/unomi-groovy-actions-services/${project.version}</bundle> - <bundle start-level="85" start="false">mvn:org.apache.unomi/unomi-groovy-actions-rest/${project.version}</bundle> <bundle start-level="85">wrap:mvn:io.github.http-builder-ng/http-builder-ng-core/1.0.4</bundle> <bundle start-level="85">mvn:org.jsoup/jsoup/1.13.1</bundle> <bundle start-level="85">mvn:com.sun.activation/javax.activation/1.2.0</bundle> + <bundle start-level="85" start="false">mvn:org.apache.unomi/unomi-groovy-actions-services/${project.version}</bundle> + <bundle start-level="85" start="false">mvn:org.apache.unomi/unomi-groovy-actions-rest/${project.version}</bundle> </feature> </features> diff --git a/extensions/groovy-actions/services/pom.xml b/extensions/groovy-actions/services/pom.xml index 8494a9938..b9b726575 100644 --- a/extensions/groovy-actions/services/pom.xml +++ b/extensions/groovy-actions/services/pom.xml @@ -104,6 +104,7 @@ <extensions>true</extensions> <configuration> <instructions> + <_dsannotations>*</_dsannotations> <Embed-Dependency>*;scope=compile|runtime</Embed-Dependency> <Unomi-Source-Folders>${project.basedir}</Unomi-Source-Folders> <DynamicImport-Package>*</DynamicImport-Package> diff --git a/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/GroovyActionDispatcher.java b/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/GroovyActionDispatcher.java index dadeabb44..f5276ab37 100644 --- a/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/GroovyActionDispatcher.java +++ b/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/GroovyActionDispatcher.java @@ -25,6 +25,8 @@ import org.apache.unomi.api.actions.ActionDispatcher; import org.apache.unomi.groovy.actions.services.GroovyActionsService; import org.apache.unomi.metrics.MetricAdapter; import org.apache.unomi.metrics.MetricsService; +import org.osgi.service.component.annotations.Component; +import org.osgi.service.component.annotations.Reference; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,6 +34,7 @@ import org.slf4j.LoggerFactory; * An implementation of an ActionDispatcher for the Groovy language. This dispatcher will load the groovy action script matching to an * actionName. If a script if found, it will be executed. */ +@Component(service = ActionDispatcher.class) public class GroovyActionDispatcher implements ActionDispatcher { private static final Logger logger = LoggerFactory.getLogger(GroovyActionDispatcher.class.getName()); @@ -39,13 +42,14 @@ public class GroovyActionDispatcher implements ActionDispatcher { private static final String GROOVY_PREFIX = "groovy"; private MetricsService metricsService; - private GroovyActionsService groovyActionsService; + @Reference public void setMetricsService(MetricsService metricsService) { this.metricsService = metricsService; } + @Reference public void setGroovyActionsService(GroovyActionsService groovyActionsService) { this.groovyActionsService = groovyActionsService; } diff --git a/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/listener/GroovyActionListener.java b/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/listener/GroovyActionListener.java index 2ac3e37c4..1dacf8af4 100644 --- a/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/listener/GroovyActionListener.java +++ b/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/listener/GroovyActionListener.java @@ -16,16 +16,17 @@ */ package org.apache.unomi.groovy.actions.listener; -import groovy.util.GroovyScriptEngine; import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.IOUtils; -import org.apache.unomi.groovy.actions.GroovyAction; import org.apache.unomi.groovy.actions.services.GroovyActionsService; -import org.apache.unomi.persistence.spi.PersistenceService; import org.osgi.framework.Bundle; import org.osgi.framework.BundleContext; import org.osgi.framework.BundleEvent; import org.osgi.framework.SynchronousBundleListener; +import org.osgi.service.component.annotations.Activate; +import org.osgi.service.component.annotations.Component; +import org.osgi.service.component.annotations.Deactivate; +import org.osgi.service.component.annotations.Reference; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,30 +40,24 @@ import java.util.Enumeration; * The description of the action will be loaded from the ActionDescriptor annotation present in the groovy file. * The script will be stored in the ES index groovyAction */ +@Component(service = SynchronousBundleListener.class) public class GroovyActionListener implements SynchronousBundleListener { private static final Logger logger = LoggerFactory.getLogger(GroovyActionListener.class.getName()); public static final String ENTRIES_LOCATION = "META-INF/cxs/actions"; - private PersistenceService persistenceService; private GroovyActionsService groovyActionsService; private BundleContext bundleContext; - public void setPersistenceService(PersistenceService persistenceService) { - this.persistenceService = persistenceService; - } - + @Reference public void setGroovyActionsService(GroovyActionsService groovyActionsService) { this.groovyActionsService = groovyActionsService; } - public void setBundleContext(BundleContext bundleContext) { + @Activate + public void postConstruct(BundleContext bundleContext) { this.bundleContext = bundleContext; - } - - public void postConstruct() { logger.debug("postConstruct {}", bundleContext.getBundle()); - createIndex(); loadGroovyActions(bundleContext); for (Bundle bundle : bundleContext.getBundles()) { if (bundle.getBundleContext() != null && bundle.getBundleId() != bundleContext.getBundle().getBundleId()) { @@ -74,6 +69,7 @@ public class GroovyActionListener implements SynchronousBundleListener { logger.info("Groovy Action Dispatcher initialized."); } + @Deactivate public void preDestroy() { processBundleStop(bundleContext); bundleContext.removeBundleListener(this); @@ -107,14 +103,6 @@ public class GroovyActionListener implements SynchronousBundleListener { } } - public void createIndex() { - if (persistenceService.createIndex(GroovyAction.ITEM_TYPE)) { - logger.info("GroovyAction index created"); - } else { - logger.info("GroovyAction index already exists"); - } - } - private void addGroovyAction(URL groovyActionURL) { try { groovyActionsService.save(FilenameUtils.getName(groovyActionURL.getPath()).replace(".groovy", ""), diff --git a/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/services/impl/GroovyActionsServiceImpl.java b/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/services/impl/GroovyActionsServiceImpl.java index b72a089b4..5761cc1c5 100644 --- a/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/services/impl/GroovyActionsServiceImpl.java +++ b/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/services/impl/GroovyActionsServiceImpl.java @@ -35,7 +35,9 @@ import org.codehaus.groovy.control.CompilerConfiguration; import org.codehaus.groovy.control.customizers.ImportCustomizer; import org.osgi.framework.BundleContext; import org.osgi.framework.wiring.BundleWiring; -import org.osgi.service.component.annotations.Reference; +import org.osgi.service.component.annotations.*; +import org.osgi.service.metatype.annotations.Designate; +import org.osgi.service.metatype.annotations.ObjectClassDefinition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,8 +57,15 @@ import static java.util.Arrays.asList; /** * Implementation of the GroovyActionService. Allows to create a groovy action from a groovy file */ +@Component(service = GroovyActionsService.class, configurationPid = "org.apache.unomi.groovy.actions") +@Designate(ocd = GroovyActionsServiceImpl.GroovyActionsServiceConfig.class) public class GroovyActionsServiceImpl implements GroovyActionsService { + @ObjectClassDefinition(name = "Groovy actions service config", description = "The configuration for the Groovy actions service") + public @interface GroovyActionsServiceConfig { + int services_groovy_actions_refresh_interval() default 1000; + } + private BundleContext bundleContext; private GroovyScriptEngine groovyScriptEngine; @@ -70,41 +79,30 @@ public class GroovyActionsServiceImpl implements GroovyActionsService { private static final Logger logger = LoggerFactory.getLogger(GroovyActionsServiceImpl.class.getName()); private static final String BASE_SCRIPT_NAME = "BaseScript"; + private static final String GROOVY_SOURCE_CODE_ID_SUFFIX = "-groovySourceCode"; - public void setBundleContext(BundleContext bundleContext) { - this.bundleContext = bundleContext; - } - - @Reference private DefinitionsService definitionsService; - - @Reference private PersistenceService persistenceService; - - @Reference private SchedulerService schedulerService; - - @Reference private ActionExecutorDispatcher actionExecutorDispatcher; + private GroovyActionsServiceConfig config; - private Integer groovyActionsRefreshInterval = 1000; - + @Reference public void setDefinitionsService(DefinitionsService definitionsService) { this.definitionsService = definitionsService; } + @Reference public void setPersistenceService(PersistenceService persistenceService) { this.persistenceService = persistenceService; } - public void setGroovyActionsRefreshInterval(Integer groovyActionsRefreshInterval) { - this.groovyActionsRefreshInterval = groovyActionsRefreshInterval; - } - + @Reference public void setSchedulerService(SchedulerService schedulerService) { this.schedulerService = schedulerService; } + @Reference public void setActionExecutorDispatcher(ActionExecutorDispatcher actionExecutorDispatcher) { this.actionExecutorDispatcher = actionExecutorDispatcher; } @@ -113,13 +111,17 @@ public class GroovyActionsServiceImpl implements GroovyActionsService { return groovyShell; } - public void postConstruct() { + @Activate + public void start(GroovyActionsServiceConfig config, BundleContext bundleContext) { logger.debug("postConstruct {}", bundleContext.getBundle()); - groovyCodeSourceMap = new HashMap<>(); - GroovyBundleResourceConnector bundleResourceConnector = new GroovyBundleResourceConnector(bundleContext); + this.config = config; + this.bundleContext = bundleContext; + this.groovyCodeSourceMap = new HashMap<>(); + + GroovyBundleResourceConnector bundleResourceConnector = new GroovyBundleResourceConnector(bundleContext); GroovyClassLoader groovyLoader = new GroovyClassLoader(bundleContext.getBundle().adapt(BundleWiring.class).getClassLoader()); - groovyScriptEngine = new GroovyScriptEngine(bundleResourceConnector, groovyLoader); + this.groovyScriptEngine = new GroovyScriptEngine(bundleResourceConnector, groovyLoader); initializeGroovyShell(); try { @@ -131,9 +133,12 @@ public class GroovyActionsServiceImpl implements GroovyActionsService { logger.info("Groovy action service initialized."); } + @Deactivate public void onDestroy() { logger.debug("onDestroy Method called"); - scheduledFuture.cancel(true); + if (scheduledFuture != null && !scheduledFuture.isCancelled()) { + scheduledFuture.cancel(true); + } } /** @@ -153,9 +158,7 @@ public class GroovyActionsServiceImpl implements GroovyActionsService { return; } logger.debug("Found Groovy base script at {}, loading... ", groovyBaseScriptURL.getPath()); - GroovyCodeSource groovyCodeSource = new GroovyCodeSource(IOUtils.toString(groovyBaseScriptURL.openStream()), BASE_SCRIPT_NAME, - "/groovy/script"); - groovyCodeSourceMap.put(BASE_SCRIPT_NAME, groovyCodeSource); + GroovyCodeSource groovyCodeSource = new GroovyCodeSource(IOUtils.toString(groovyBaseScriptURL.openStream()), BASE_SCRIPT_NAME, "/groovy/script"); groovyScriptEngine.getGroovyClassLoader().parseClass(groovyCodeSource, true); } @@ -183,15 +186,10 @@ public class GroovyActionsServiceImpl implements GroovyActionsService { @Override public void save(String actionName, String groovyScript) { - handleFile(actionName, groovyScript); - } - - private void handleFile(String actionName, String groovyScript) { GroovyCodeSource groovyCodeSource = buildClassScript(groovyScript, actionName); try { saveActionType(groovyShell.parse(groovyCodeSource).getClass().getMethod("execute").getAnnotation(Action.class)); saveScript(actionName, groovyScript); - groovyCodeSourceMap.put(actionName, groovyCodeSource); logger.info("The script {} has been loaded.", actionName); } catch (NoSuchMethodException e) { logger.error("Failed to save the script {}", actionName, e); @@ -219,19 +217,21 @@ public class GroovyActionsServiceImpl implements GroovyActionsService { @Override public void remove(String id) { - try { - definitionsService.removeActionType( - groovyShell.parse(groovyCodeSourceMap.get(id)).getClass().getMethod("execute").getAnnotation(Action.class).id()); - } catch (NoSuchMethodException e) { - logger.error("Failed to delete the action type for the id {}", id, e); + String groovySourceCodeId = getGroovyCodeSourceIdForActionId(id); + if (groovyCodeSourceMap.containsKey(groovySourceCodeId)) { + try { + definitionsService.removeActionType( + groovyShell.parse(groovyCodeSourceMap.get(groovySourceCodeId)).getClass().getMethod("execute").getAnnotation(Action.class).id()); + } catch (NoSuchMethodException e) { + logger.error("Failed to delete the action type for the id {}", id, e); + } + persistenceService.remove(groovySourceCodeId, GroovyAction.class); } - persistenceService.remove(id, GroovyAction.class); - groovyCodeSourceMap.remove(id); } @Override public GroovyCodeSource getGroovyCodeSource(String id) { - return groovyCodeSourceMap.get(id); + return groovyCodeSourceMap.get(getGroovyCodeSourceIdForActionId(id)); } /** @@ -245,15 +245,25 @@ public class GroovyActionsServiceImpl implements GroovyActionsService { return new GroovyCodeSource(groovyScript, actionName, "/groovy/script"); } - private void saveScript(String name, String script) { - GroovyAction groovyScript = new GroovyAction(name, script); + /** + * We use a suffix for avoiding id conflict between the actionType and the groovyAction in ElasticSearch + * Since those items are now stored in the same ES index + * @param actionName name/id of the actionType + * @return id of the groovyAction source code for query/save/storage usage. + */ + private String getGroovyCodeSourceIdForActionId(String actionName) { + return actionName + GROOVY_SOURCE_CODE_ID_SUFFIX; + } + + private void saveScript(String actionName, String script) { + String groovyName = getGroovyCodeSourceIdForActionId(actionName); + GroovyAction groovyScript = new GroovyAction(groovyName, script); persistenceService.save(groovyScript); + logger.info("The script {} has been persisted.", groovyName); } private void refreshGroovyActions() { Map<String, GroovyCodeSource> refreshedGroovyCodeSourceMap = new HashMap<>(); - GroovyCodeSource baseScript = groovyCodeSourceMap.get(BASE_SCRIPT_NAME); - refreshedGroovyCodeSourceMap.put(BASE_SCRIPT_NAME, baseScript); persistenceService.getAllItems(GroovyAction.class).forEach(groovyAction -> refreshedGroovyCodeSourceMap .put(groovyAction.getName(), buildClassScript(groovyAction.getScript(), groovyAction.getName()))); groovyCodeSourceMap = refreshedGroovyCodeSourceMap; @@ -266,7 +276,7 @@ public class GroovyActionsServiceImpl implements GroovyActionsService { refreshGroovyActions(); } }; - scheduledFuture = schedulerService.getScheduleExecutorService().scheduleWithFixedDelay(task, 0, groovyActionsRefreshInterval, + scheduledFuture = schedulerService.getScheduleExecutorService().scheduleWithFixedDelay(task, 0, config.services_groovy_actions_refresh_interval(), TimeUnit.MILLISECONDS); } } diff --git a/extensions/groovy-actions/services/src/main/resources/META-INF/cxs/mappings/groovyAction.json b/extensions/groovy-actions/services/src/main/resources/META-INF/cxs/mappings/groovyAction.json deleted file mode 100644 index 6cd8d3ff8..000000000 --- a/extensions/groovy-actions/services/src/main/resources/META-INF/cxs/mappings/groovyAction.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "dynamic_templates": [ - { - "all": { - "match": "*", - "match_mapping_type": "string", - "mapping": { - "type": "text", - "analyzer": "folding", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - } - ], - "properties": { - "name": { - "type": "text" - }, - "script": { - "type": "text" - } - } -} \ No newline at end of file diff --git a/extensions/groovy-actions/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/extensions/groovy-actions/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml deleted file mode 100644 index 433cd549c..000000000 --- a/extensions/groovy-actions/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml +++ /dev/null @@ -1,69 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one or more - ~ contributor license agreements. See the NOTICE file distributed with - ~ this work for additional information regarding copyright ownership. - ~ The ASF licenses this file to You under the Apache License, Version 2.0 - ~ (the "License"); you may not use this file except in compliance with - ~ the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - --> - -<blueprint xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xmlns:cm="http://aries.apache.org/blueprint/xmlns/blueprint-cm/v1.1.0" - xmlns="http://www.osgi.org/xmlns/blueprint/v1.0.0" - xsi:schemaLocation="http://www.osgi.org/xmlns/blueprint/v1.0.0 http://www.osgi.org/xmlns/blueprint/v1.0.0/blueprint.xsd"> - - <cm:property-placeholder persistent-id="org.apache.unomi.groovy.actions" update-strategy="reload"> - <cm:default-properties> - <cm:property name="services.groovy.actions.refresh.interval" value="1000"/> - </cm:default-properties> - </cm:property-placeholder> - - <reference id="metricsService" interface="org.apache.unomi.metrics.MetricsService"/> - <reference id="definitionsService" interface="org.apache.unomi.api.services.DefinitionsService"/> - <reference id="persistenceService" interface="org.apache.unomi.persistence.spi.PersistenceService"/> - <reference id="schedulerService" interface="org.apache.unomi.api.services.SchedulerService"/> - <reference id="actionExecutorDispatcher" interface="org.apache.unomi.services.actions.ActionExecutorDispatcher"/> - - <bean id="groovyActionsServiceImpl" class="org.apache.unomi.groovy.actions.services.impl.GroovyActionsServiceImpl" - init-method="postConstruct" destroy-method="onDestroy"> - <property name="bundleContext" ref="blueprintBundleContext"/> - <property name="definitionsService" ref="definitionsService"/> - <property name="persistenceService" ref="persistenceService"/> - <property name="schedulerService" ref="schedulerService"/> - <property name="actionExecutorDispatcher" ref="actionExecutorDispatcher"/> - <property name="groovyActionsRefreshInterval" value="${services.groovy.actions.refresh.interval}"/> - </bean> - <service id="groovyActionsService" ref="groovyActionsServiceImpl" - interface="org.apache.unomi.groovy.actions.services.GroovyActionsService"/> - - <bean id="groovyActionDispatcherImpl" class="org.apache.unomi.groovy.actions.GroovyActionDispatcher"> - <property name="metricsService" ref="metricsService"/> - <property name="groovyActionsService" ref="groovyActionsServiceImpl"/> - </bean> - <service id="groovyActionDispatcher" ref="groovyActionDispatcherImpl"> - <interfaces> - <value>org.apache.unomi.api.actions.ActionDispatcher</value> - </interfaces> - </service> - - <bean id="groovyActionListenerImpl" class="org.apache.unomi.groovy.actions.listener.GroovyActionListener" - init-method="postConstruct" destroy-method="preDestroy"> - <property name="persistenceService" ref="persistenceService"/> - <property name="bundleContext" ref="blueprintBundleContext"/> - <property name="groovyActionsService" ref="groovyActionsServiceImpl"/> - </bean> - <service id="groovyActionListener" ref="groovyActionListenerImpl"> - <interfaces> - <value>org.osgi.framework.SynchronousBundleListener</value> - </interfaces> - </service> -</blueprint> diff --git a/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/impl/SchemaServiceImpl.java b/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/impl/SchemaServiceImpl.java index ebc431f7f..44ff90f7b 100644 --- a/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/impl/SchemaServiceImpl.java +++ b/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/impl/SchemaServiceImpl.java @@ -296,14 +296,6 @@ public class SchemaServiceImpl implements SchemaService { return schema; } - private void initPersistenceIndex() { - if (persistenceService.createIndex(JsonSchemaWrapper.ITEM_TYPE)) { - logger.info("{} index created", JsonSchemaWrapper.ITEM_TYPE); - } else { - logger.info("{} index already exists", JsonSchemaWrapper.ITEM_TYPE); - } - } - private void initTimers() { TimerTask task = new TimerTask() { @Override @@ -345,7 +337,6 @@ public class SchemaServiceImpl implements SchemaService { public void init() { scheduler = Executors.newSingleThreadScheduledExecutor(); - initPersistenceIndex(); initJsonSchemaFactory(); initTimers(); logger.info("Schema service initialized."); diff --git a/itests/pom.xml b/itests/pom.xml index f0a602efe..375e816b6 100644 --- a/itests/pom.xml +++ b/itests/pom.xml @@ -133,6 +133,12 @@ <version>${project.version}</version> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.unomi</groupId> + <artifactId>unomi-groovy-actions-services</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> <dependency> <groupId>org.codehaus.groovy</groupId> <artifactId>groovy</artifactId> diff --git a/itests/src/test/java/org/apache/unomi/itests/GroovyActionsServiceIT.java b/itests/src/test/java/org/apache/unomi/itests/GroovyActionsServiceIT.java index 20c507f8d..7fecea183 100644 --- a/itests/src/test/java/org/apache/unomi/itests/GroovyActionsServiceIT.java +++ b/itests/src/test/java/org/apache/unomi/itests/GroovyActionsServiceIT.java @@ -101,10 +101,10 @@ public class GroovyActionsServiceIT extends BaseIT { groovyActionsService.save(UPDATE_ADDRESS_ACTION, loadGroovyAction(UPDATE_ADDRESS_ACTION_GROOVY_FILE)); keepTrying("Failed waiting for the creation of the GroovyAction for the trigger action test", - () -> groovyActionsService.getGroovyCodeSource(UPDATE_ADDRESS_ACTION), Objects::nonNull, 1000, 100); + () -> groovyActionsService.getGroovyCodeSource(UPDATE_ADDRESS_ACTION), Objects::nonNull, DEFAULT_TRYING_TIMEOUT, DEFAULT_TRYING_TRIES); ActionType actionType = keepTrying("Failed waiting for the creation of the GroovyAction for trigger action test", - () -> definitionsService.getActionType(UPDATE_ADDRESS_GROOVY_ACTION), Objects::nonNull, 1000, 100); + () -> definitionsService.getActionType(UPDATE_ADDRESS_GROOVY_ACTION), Objects::nonNull, DEFAULT_TRYING_TIMEOUT, DEFAULT_TRYING_TRIES); Assert.assertNotNull(actionType); @@ -118,9 +118,12 @@ public class GroovyActionsServiceIT extends BaseIT { groovyActionsService.save(UPDATE_ADDRESS_ACTION, loadGroovyAction(UPDATE_ADDRESS_ACTION_GROOVY_FILE)); ActionType actionType = keepTrying("Failed waiting for the creation of the GroovyAction for the save test", - () -> definitionsService.getActionType(UPDATE_ADDRESS_GROOVY_ACTION), Objects::nonNull, 1000, 100); + () -> definitionsService.getActionType(UPDATE_ADDRESS_GROOVY_ACTION), Objects::nonNull, DEFAULT_TRYING_TIMEOUT, DEFAULT_TRYING_TRIES); - Assert.assertEquals(UPDATE_ADDRESS_ACTION, groovyActionsService.getGroovyCodeSource(UPDATE_ADDRESS_ACTION).getName()); + GroovyCodeSource groovyCodeSource = keepTrying("Failed waiting for the creation of the GroovyAction for the save test", + () -> groovyActionsService.getGroovyCodeSource(UPDATE_ADDRESS_ACTION), Objects::nonNull, DEFAULT_TRYING_TIMEOUT, DEFAULT_TRYING_TRIES); + + Assert.assertEquals(UPDATE_ADDRESS_ACTION + "-groovySourceCode", groovyActionsService.getGroovyCodeSource(UPDATE_ADDRESS_ACTION).getName()); Assert.assertTrue(actionType.getMetadata().getId().contains(UPDATE_ADDRESS_GROOVY_ACTION)); Assert.assertEquals(2, actionType.getMetadata().getSystemTags().size()); @@ -137,7 +140,7 @@ public class GroovyActionsServiceIT extends BaseIT { groovyActionsService.save(UPDATE_ADDRESS_ACTION, loadGroovyAction(UPDATE_ADDRESS_ACTION_GROOVY_FILE)); GroovyCodeSource groovyCodeSource = keepTrying("Failed waiting for the creation of the GroovyAction for the remove test", - () -> groovyActionsService.getGroovyCodeSource(UPDATE_ADDRESS_ACTION), Objects::nonNull, 1000, 100); + () -> groovyActionsService.getGroovyCodeSource(UPDATE_ADDRESS_ACTION), Objects::nonNull, DEFAULT_TRYING_TIMEOUT, DEFAULT_TRYING_TRIES); Assert.assertNotNull(groovyCodeSource); diff --git a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java index a3e320a3c..8ae93daf4 100644 --- a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java +++ b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java @@ -40,6 +40,7 @@ public class Migrate16xTo220IT extends BaseIT { private int sessionCount = 0; private static final int NUMBER_DUPLICATE_SESSIONS = 3; + private static final int NUMBER_PERSONA_SESSIONS = 2; @Override @Before public void waitForStartup() throws InterruptedException { @@ -96,6 +97,9 @@ public class Migrate16xTo220IT extends BaseIT { /** * Checks if at least the new index for events and sessions exists. + * Also checks: + * - duplicated sessions are correctly removed (-3 sessions in final count) + * - persona sessions are now merged in session index due to index reduction in 2_2_0 (+2 sessions in final count) */ private void checkEventSessionRollover2_2_0() throws IOException { Assert.assertTrue(MigrationUtils.indexExists(httpClient, "http://localhost:9400", "context-event-000001")); @@ -113,23 +117,20 @@ public class Migrate16xTo220IT extends BaseIT { newSessioncount += jsonNode.get("count").asInt(); } Assert.assertEquals(eventCount, newEventcount); - Assert.assertEquals(sessionCount - NUMBER_DUPLICATE_SESSIONS, newSessioncount); + Assert.assertEquals(sessionCount - NUMBER_DUPLICATE_SESSIONS + NUMBER_PERSONA_SESSIONS, newSessioncount); } /** * Multiple index mappings have been update, check a simple check that after migration those mappings contains the latest modifications. */ private void checkForMappingUpdates() throws IOException { - Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-scope/_mapping", null).contains("\"match\":\"*\",\"match_mapping_type\":\"string\",\"mapping\":{\"analyzer\":\"folding\"")); - Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-profilealias/_mapping", null).contains("\"match\":\"*\",\"match_mapping_type\":\"string\",\"mapping\":{\"analyzer\":\"folding\"")); - Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-segment/_mapping", null).contains("\"condition\":{\"type\":\"object\",\"enabled\":false}")); - Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-scoring/_mapping", null).contains("\"condition\":{\"type\":\"object\",\"enabled\":false}")); - Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-campaign/_mapping", null).contains("\"entryCondition\":{\"type\":\"object\",\"enabled\":false}")); - Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-conditiontype/_mapping", null).contains("\"parentCondition\":{\"type\":\"object\",\"enabled\":false}")); - Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-goal/_mapping", null).contains("\"startEvent\":{\"type\":\"object\",\"enabled\":false}")); - Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-patch/_mapping", null).contains("\"data\":{\"type\":\"object\",\"enabled\":false}")); - Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-rule/_mapping", null).contains("\"condition\":{\"type\":\"object\",\"enabled\":false}")); - Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-rule/_mapping", null).contains("\"parameterValues\":{\"type\":\"object\",\"enabled\":false}")); + Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-systemitems/_mapping", null).contains("\"match\":\"*\",\"match_mapping_type\":\"string\",\"mapping\":{\"analyzer\":\"folding\"")); + Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-systemitems/_mapping", null).contains("\"condition\":{\"type\":\"object\",\"enabled\":false}")); + Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-systemitems/_mapping", null).contains("\"entryCondition\":{\"type\":\"object\",\"enabled\":false}")); + Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-systemitems/_mapping", null).contains("\"parentCondition\":{\"type\":\"object\",\"enabled\":false}")); + Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-systemitems/_mapping", null).contains("\"startEvent\":{\"type\":\"object\",\"enabled\":false}")); + Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-systemitems/_mapping", null).contains("\"data\":{\"type\":\"object\",\"enabled\":false}")); + Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-systemitems/_mapping", null).contains("\"parameterValues\":{\"type\":\"object\",\"enabled\":false}")); Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-profile/_mapping", null).contains("\"interests\":{\"type\":\"nested\"")); for (String eventIndex : MigrationUtils.getIndexesPrefixedBy(httpClient, "http://localhost:9400", "context-event-")) { Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/" + eventIndex + "/_mapping", null).contains("\"flattenedProperties\":{\"type\":\"flattened\"}")); diff --git a/kar/pom.xml b/kar/pom.xml index 6e41b8097..3e72e3994 100644 --- a/kar/pom.xml +++ b/kar/pom.xml @@ -124,16 +124,6 @@ <artifactId>unomi-json-schema-rest</artifactId> <version>${project.version}</version> </dependency> - <dependency> - <groupId>org.apache.unomi</groupId> - <artifactId>unomi-groovy-actions-services</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.unomi</groupId> - <artifactId>unomi-groovy-actions-rest</artifactId> - <version>${project.version}</version> - </dependency> <dependency> <groupId>org.apache.unomi</groupId> <artifactId>cxs-privacy-extension-services</artifactId> diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java index ecc703511..13a802db0 100644 --- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java +++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java @@ -254,6 +254,34 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, private Map<String, Map<String, Map<String, Object>>> knownMappings = new HashMap<>(); + private static final Map<String, String> itemTypeIndexNameMap = new HashMap<>(); + static { + itemTypeIndexNameMap.put("actionType", "systemItems"); + itemTypeIndexNameMap.put("campaign", "systemItems"); + itemTypeIndexNameMap.put("campaignevent", "systemItems"); + itemTypeIndexNameMap.put("goal", "systemItems"); + itemTypeIndexNameMap.put("userList", "systemItems"); + itemTypeIndexNameMap.put("propertyType", "systemItems"); + itemTypeIndexNameMap.put("scope", "systemItems"); + itemTypeIndexNameMap.put("conditionType", "systemItems"); + itemTypeIndexNameMap.put("rule", "systemItems"); + itemTypeIndexNameMap.put("scoring", "systemItems"); + itemTypeIndexNameMap.put("segment", "systemItems"); + itemTypeIndexNameMap.put("groovyAction", "systemItems"); + itemTypeIndexNameMap.put("topic", "systemItems"); + itemTypeIndexNameMap.put("patch", "systemItems"); + itemTypeIndexNameMap.put("jsonSchema", "systemItems"); + itemTypeIndexNameMap.put("importConfig", "systemItems"); + itemTypeIndexNameMap.put("exportConfig", "systemItems"); + itemTypeIndexNameMap.put("rulestats", "systemItems"); + + itemTypeIndexNameMap.put("profile", "profile"); + itemTypeIndexNameMap.put("persona", "profile"); + + itemTypeIndexNameMap.put("session", "session"); + itemTypeIndexNameMap.put("personaSession", "session"); + } + public void setBundleContext(BundleContext bundleContext) { this.bundleContext = bundleContext; } @@ -1099,13 +1127,14 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, for (int i = 0; i < scripts.length; i++) { RefreshRequest refreshRequest = new RefreshRequest(index); client.indices().refresh(refreshRequest, RequestOptions.DEFAULT); + QueryBuilder queryBuilder = conditionESQueryBuilderDispatcher.buildFilter(conditions[i]); UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(index); updateByQueryRequest.setConflicts("proceed"); updateByQueryRequest.setMaxRetries(1000); updateByQueryRequest.setSlices(2); updateByQueryRequest.setScript(scripts[i]); - updateByQueryRequest.setQuery(conditionESQueryBuilderDispatcher.buildFilter(conditions[i])); + updateByQueryRequest.setQuery(isItemTypeSharingIndex(itemType) ? wrapWithItemTypeQuery(itemType, queryBuilder) : queryBuilder); BulkByScrollResponse response = client.updateByQuery(updateByQueryRequest, RequestOptions.DEFAULT); @@ -1264,8 +1293,9 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, protected Boolean execute(Object... args) throws Exception { try { String itemType = Item.getItemType(clazz); + QueryBuilder queryBuilder = conditionESQueryBuilderDispatcher.getQueryBuilder(query); final DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(getIndexNameForQuery(itemType)) - .setQuery(conditionESQueryBuilderDispatcher.getQueryBuilder(query)) + .setQuery(isItemTypeSharingIndex(itemType) ? wrapWithItemTypeQuery(itemType, queryBuilder) : queryBuilder) // Setting slices to auto will let Elasticsearch choose the number of slices to use. // This setting will use one slice per shard, up to a certain limit. // The delete request will be more efficient and faster than no slicing. @@ -1903,7 +1933,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, CountRequest countRequest = new CountRequest(getIndexNameForQuery(itemType)); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(filter); + searchSourceBuilder.query(isItemTypeSharingIndex(itemType) ? wrapWithItemTypeQuery(itemType, filter) : filter); countRequest.source(searchSourceBuilder); CountResponse response = client.count(countRequest, RequestOptions.DEFAULT); return response.getCount(); @@ -1938,7 +1968,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() .fetchSource(true) .seqNoAndPrimaryTerm(true) - .query(query) + .query(isItemTypeSharingIndex(itemType) ? wrapWithItemTypeQuery(itemType, query) : query) .size(size < 0 ? defaultQueryLimit : size) .from(offset); if (scrollTimeValidity != null) { @@ -2159,7 +2189,9 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, SearchRequest searchRequest = new SearchRequest(getIndexNameForQuery(itemType)); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.size(0); - searchSourceBuilder.query(QueryBuilders.matchAllQuery()); + MatchAllQueryBuilder matchAll = QueryBuilders.matchAllQuery(); + boolean isItemTypeSharingIndex = isItemTypeSharingIndex(itemType); + searchSourceBuilder.query(isItemTypeSharingIndex ? getItemTypeQueryBuilder(itemType) : matchAll); List<AggregationBuilder> lastAggregation = new ArrayList<AggregationBuilder>(); if (aggregate != null) { @@ -2240,11 +2272,15 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, } if (filter != null) { - searchSourceBuilder.query(conditionESQueryBuilderDispatcher.buildFilter(filter)); + searchSourceBuilder.query(isItemTypeSharingIndex ? + wrapWithItemTypeQuery(itemType, conditionESQueryBuilderDispatcher.buildFilter(filter)) : + conditionESQueryBuilderDispatcher.buildFilter(filter)); } } else { if (filter != null) { - AggregationBuilder filterAggregation = AggregationBuilders.filter("filter", conditionESQueryBuilderDispatcher.buildFilter(filter)); + AggregationBuilder filterAggregation = AggregationBuilders.filter("filter", isItemTypeSharingIndex ? + wrapWithItemTypeQuery(itemType, conditionESQueryBuilderDispatcher.buildFilter(filter)) : + conditionESQueryBuilderDispatcher.buildFilter(filter)); for (AggregationBuilder aggregationBuilder : lastAggregation) { filterAggregation.subAggregation(aggregationBuilder); } @@ -2465,7 +2501,8 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, SearchRequest searchRequest = new SearchRequest(getIndexNameForQuery(itemType)); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() .size(0) - .query(QueryBuilders.matchAllQuery()); + .query(isItemTypeSharingIndex(itemType) ? getItemTypeQueryBuilder(itemType) : QueryBuilders.matchAllQuery()); + AggregationBuilder filterAggregation = AggregationBuilders.filter("metrics", conditionESQueryBuilderDispatcher.buildFilter(condition)); if (metrics != null) { @@ -2598,13 +2635,32 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, private String getIndexNameForQuery(String itemType) { return isItemTypeRollingOver(itemType) ? getRolloverIndexForQuery(itemType) : getIndex(itemType); } - + private String getRolloverIndexForQuery(String itemType) { return indexPrefix + "-" + itemType.toLowerCase() + "-*"; } - private String getIndex(String indexItemTypePart) { - return (indexPrefix + "-" + indexItemTypePart).toLowerCase(); + private String getIndex(String itemType) { + return (indexPrefix + "-" + getIndexNameForItemType(itemType)).toLowerCase(); + } + + private String getIndexNameForItemType(String itemType) { + return itemTypeIndexNameMap.getOrDefault(itemType, itemType); + } + + private QueryBuilder wrapWithItemTypeQuery(String itemType, QueryBuilder originalQuery) { + BoolQueryBuilder wrappedQuery = QueryBuilders.boolQuery(); + wrappedQuery.must(getItemTypeQueryBuilder(itemType)); + wrappedQuery.must(originalQuery); + return wrappedQuery; + } + + private QueryBuilder getItemTypeQueryBuilder(String itemType) { + return QueryBuilders.termQuery("itemType", ConditionContextHelper.foldToASCII(itemType)); + } + + private boolean isItemTypeSharingIndex(String itemType) { + return itemTypeIndexNameMap.containsKey(itemType); } private boolean isItemTypeRollingOver(String itemType) { diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/actionType.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/actionType.json deleted file mode 100644 index e1ac5f8d4..000000000 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/actionType.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "dynamic_templates": [ - { - "all": { - "match": "*", - "match_mapping_type": "string", - "mapping": { - "type": "text", - "analyzer": "folding", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json deleted file mode 100644 index 7d6a564d0..000000000 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "dynamic_templates": [ - { - "all": { - "match": "*", - "match_mapping_type": "string", - "mapping": { - "type": "text", - "analyzer": "folding", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - } - ], - "properties": { - "cost": { - "type": "double" - }, - "eventDate": { - "type": "date" - }, - "metadata": { - "properties": { - "enabled": { - "type": "boolean" - }, - "hidden": { - "type": "boolean" - }, - "missingPlugins": { - "type": "boolean" - }, - "readOnly": { - "type": "boolean" - } - } - } - } -} \ No newline at end of file diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/exportConfig.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/exportConfig.json deleted file mode 100644 index e1ac5f8d4..000000000 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/exportConfig.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "dynamic_templates": [ - { - "all": { - "match": "*", - "match_mapping_type": "string", - "mapping": { - "type": "text", - "analyzer": "folding", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/importConfig.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/importConfig.json deleted file mode 100644 index e1ac5f8d4..000000000 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/importConfig.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "dynamic_templates": [ - { - "all": { - "match": "*", - "match_mapping_type": "string", - "mapping": { - "type": "text", - "analyzer": "folding", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/jsonschema.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/jsonschema.json deleted file mode 100644 index 896e1be21..000000000 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/jsonschema.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "dynamic_templates": [ - { - "all": { - "match": "*", - "match_mapping_type": "string", - "mapping": { - "type": "text", - "analyzer": "folding", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - } - ], - "properties": { - "schema": { - "type": "text" - } - } -} diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/persona.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/persona.json deleted file mode 100644 index d0142451f..000000000 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/persona.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dynamic_templates": [ - { - "all": { - "match": "*", - "match_mapping_type": "string", - "mapping": { - "type": "text", - "analyzer": "folding", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - } - ], - "properties": { - "properties": { - "properties": { - "firstVisit": { - "type": "date" - }, - "lastVisit": { - "type": "date" - }, - "previousVisit": { - "type": "date" - }, - "nbOfVisits": { - "type": "long" - } - } - } - } -} \ No newline at end of file diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json deleted file mode 100644 index c635e0285..000000000 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "dynamic_templates": [ - { - "all": { - "match": "*", - "match_mapping_type": "string", - "mapping": { - "type": "text", - "analyzer": "folding", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - } - ], - "properties": { - "duration": { - "type": "long" - }, - "timeStamp": { - "type": "date" - }, - "lastEventDate": { - "type": "date" - }, - "properties": { - "properties": { - "location": { - "type": "geo_point" - } - } - }, - "size": { - "type": "long" - } - } -} \ No newline at end of file diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json deleted file mode 100644 index 3898c8c68..000000000 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "dynamic_templates": [ - { - "all": { - "match": "*", - "match_mapping_type": "string", - "mapping": { - "type": "text", - "analyzer": "folding", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - } - ], - "properties": { - "dateRanges": { - "properties": { - } - }, - "metadata": { - "properties": { - "enabled": { - "type": "boolean" - }, - "hidden": { - "type": "boolean" - }, - "missingPlugins": { - "type": "boolean" - }, - "readOnly": { - "type": "boolean" - } - } - }, - "multivalued": { - "type": "boolean" - }, - "numericRanges": { - "properties": { - "from": { - "type": "double" - }, - "to": { - "type": "double" - } - } - }, - "protected": { - "type": "boolean" - }, - "rank": { - "type": "double" - } - } -} \ No newline at end of file diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rulestats.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rulestats.json deleted file mode 100644 index e1ac5f8d4..000000000 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rulestats.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "dynamic_templates": [ - { - "all": { - "match": "*", - "match_mapping_type": "string", - "mapping": { - "type": "text", - "analyzer": "folding", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/systemItems.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/systemItems.json new file mode 100644 index 000000000..ca5a7a397 --- /dev/null +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/systemItems.json @@ -0,0 +1,141 @@ +{ + "dynamic_templates": [ + { + "all": { + "match": "*", + "match_mapping_type": "string", + "mapping": { + "type": "text", + "analyzer": "folding", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + ], + "properties": { + "cost": { + "type": "double" + }, + "startDate": { + "type": "date" + }, + "endDate": { + "type": "date" + }, + "metadata": { + "properties": { + "enabled": { + "type": "boolean" + }, + "hidden": { + "type": "boolean" + }, + "missingPlugins": { + "type": "boolean" + }, + "readOnly": { + "type": "boolean" + } + } + }, + "entryCondition": { + "type": "object", + "enabled": false + }, + "parentCondition": { + "type": "object", + "enabled": false + }, + "startEvent": { + "type": "object", + "enabled": false + }, + "targetEvent": { + "type": "object", + "enabled": false + }, + "eventDate": { + "type": "date" + }, + "multivalued": { + "type": "boolean" + }, + "numericRanges": { + "properties": { + "from": { + "type": "double" + }, + "to": { + "type": "double" + } + } + }, + "protected": { + "type": "boolean" + }, + "rank": { + "type": "double" + }, + "dateRanges": { + "properties": { + } + }, + "priority": { + "type": "long" + }, + "raiseEventOnlyOnceForProfile": { + "type": "boolean" + }, + "raiseEventOnlyOnceForSession": { + "type": "boolean" + }, + "raiseEventOnlyOnce": { + "type": "boolean" + }, + "condition": { + "type": "object", + "enabled": false + }, + "actions": { + "properties": { + "parameterValues": { + "type": "object", + "enabled": false + } + } + }, + "elements": { + "properties": { + "condition": { + "type": "object", + "enabled": false + } + } + }, + + "patchedItemId": { + "type": "text" + }, + "patchedItemType": { + "type": "text" + }, + "operation": { + "type": "text" + }, + "data": { + "type": "object", + "enabled": false + }, + "lastApplication": { + "type": "date" + }, + "schema": { + "type": "text" + } + } +} \ No newline at end of file diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/topic.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/topic.json deleted file mode 100644 index d2d90cb94..000000000 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/topic.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "dynamic_templates": [ - { - "all": { - "match": "*", - "match_mapping_type": "string", - "mapping": { - "type": "text", - "analyzer": "folding", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - } - ] -} diff --git a/services/src/main/java/org/apache/unomi/services/impl/rules/RulesServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/rules/RulesServiceImpl.java index e0ffbee56..fafec7bcf 100644 --- a/services/src/main/java/org/apache/unomi/services/impl/rules/RulesServiceImpl.java +++ b/services/src/main/java/org/apache/unomi/services/impl/rules/RulesServiceImpl.java @@ -42,10 +42,12 @@ import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; public class RulesServiceImpl implements RulesService, EventListenerService, SynchronousBundleListener { public static final String RULE_QUERY_PREFIX = "rule_"; + private static final String RULE_STAT_ID_SUFFIX = "-stat"; public static final String TRACKED_PARAMETER = "trackedConditionParameters"; private static final Logger logger = LoggerFactory.getLogger(RulesServiceImpl.class.getName()); @@ -251,9 +253,10 @@ public class RulesServiceImpl implements RulesService, EventListenerService, Syn } private RuleStatistics getLocalRuleStatistics(Rule rule) { - RuleStatistics ruleStatistics = this.allRuleStatistics.get(rule.getItemId()); + String ruleStatisticsId = getRuleStatisticId(rule.getItemId()); + RuleStatistics ruleStatistics = this.allRuleStatistics.get(ruleStatisticsId); if (ruleStatistics == null) { - ruleStatistics = new RuleStatistics(rule.getItemId()); + ruleStatistics = new RuleStatistics(ruleStatisticsId); } return ruleStatistics; } @@ -264,6 +267,10 @@ public class RulesServiceImpl implements RulesService, EventListenerService, Syn allRuleStatistics.put(ruleStatistics.getItemId(), ruleStatistics); } + private String getRuleStatisticId(String ruleID) { + return ruleID + RULE_STAT_ID_SUFFIX; + } + public void refreshRules() { try { // we use local variables to make sure we quickly switch the collections since the refresh is called often @@ -330,21 +337,24 @@ public class RulesServiceImpl implements RulesService, EventListenerService, Syn RuleStatistics ruleStatistics = getLocalRuleStatistics(rule); ruleStatistics.setLocalExecutionCount(ruleStatistics.getLocalExecutionCount() + 1); ruleStatistics.setLocalActionsTime(ruleStatistics.getLocalActionsTime() + totalActionsTime); - this.allRuleStatistics.put(rule.getItemId(), ruleStatistics); + this.allRuleStatistics.put(ruleStatistics.getItemId(), ruleStatistics); } return changes; } @Override public RuleStatistics getRuleStatistics(String ruleId) { - if (allRuleStatistics.containsKey(ruleId)) { - return allRuleStatistics.get(ruleId); + String ruleStatisticsId = getRuleStatisticId(ruleId); + if (allRuleStatistics.containsKey(ruleStatisticsId)) { + return allRuleStatistics.get(ruleStatisticsId); } - return persistenceService.load(ruleId, RuleStatistics.class); + return persistenceService.load(ruleStatisticsId, RuleStatistics.class); } public Map<String, RuleStatistics> getAllRuleStatistics() { - return allRuleStatistics; + return allRuleStatistics.keySet().stream() + .collect(Collectors.toMap(key -> key.endsWith(RULE_STAT_ID_SUFFIX) ? + key.substring(0, key.length() - RULE_STAT_ID_SUFFIX.length()) : key, allRuleStatistics::get)); } @Override diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java index a12fd6ec7..2050d6ba5 100644 --- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java +++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java @@ -173,8 +173,8 @@ public class MigrationUtils { return baseRequest.replace("#rolloverHotActions", rolloverHotActions.toString()); } - public static void moveToIndex(CloseableHttpClient httpClient, BundleContext bundleContext, String esAddress, String sourceIndexName, String targetIndexName) throws Exception { - String reIndexRequest = resourceAsString(bundleContext, "requestBody/2.2.0/base_reindex_request.json").replace("#source", sourceIndexName).replace("#dest", targetIndexName); + public static void moveToIndex(CloseableHttpClient httpClient, BundleContext bundleContext, String esAddress, String sourceIndexName, String targetIndexName, String painlessScript) throws Exception { + String reIndexRequest = resourceAsString(bundleContext, "requestBody/2.2.0/base_reindex_request.json").replace("#source", sourceIndexName).replace("#dest", targetIndexName).replace("#painless", StringUtils.isNotEmpty(painlessScript) ? getScriptPart(painlessScript) : ""); HttpUtils.executePostRequest(httpClient, esAddress + "/_reindex", reIndexRequest, null); } @@ -225,10 +225,13 @@ public class MigrationUtils { HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexNameCloned, null); } }); - // Do a refresh - HttpUtils.executePostRequest(httpClient, esAddress + "/" + indexName + "/_refresh", null, null); - waitForYellowStatus(httpClient, esAddress, migrationContext); + migrationContext.performMigrationStep("Reindex step for: " + indexName + " (refresh at the end)", () -> { + // Do a refresh + HttpUtils.executePostRequest(httpClient, esAddress + "/" + indexName + "/_refresh", null, null); + + waitForYellowStatus(httpClient, esAddress, migrationContext); + }); } public static void scrollQuery(CloseableHttpClient httpClient, String esAddress, String queryURL, String query, String scrollDuration, ScrollCallback scrollCallback) throws IOException { diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy index dd841ff11..3d013b1e4 100644 --- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy +++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy @@ -25,7 +25,7 @@ String indexPrefix = context.getConfigString("indexPrefix") String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json") String[] indicesToReindex = ["segment", "scoring", "campaign", "conditionType", "goal", "patch", "rule"]; indicesToReindex.each { indexToReindex -> - String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "${indexToReindex}.json") + String mapping = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/mappings/${indexToReindex}.json") String newIndexSettings = MigrationUtils.buildIndexCreationRequest(baseSettings, mapping, context, false) MigrationUtils.reIndex(context.getHttpClient(), bundleContext, esAddress, "${indexPrefix}-${indexToReindex.toLowerCase()}", newIndexSettings, null, context) } diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-20-scopes.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-20-scopes.groovy index d9aa5f889..1f63d1b82 100644 --- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-20-scopes.groovy +++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-20-scopes.groovy @@ -32,7 +32,7 @@ String scopeIndex = indexPrefix + "-scope" context.performMigrationStep("2.0.0-create-scope-index", () -> { if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, scopeIndex)) { String baseRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json") - String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "scope.json") + String mapping = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/mappings/scope.json") String newIndexSettings = MigrationUtils.buildIndexCreationRequest(baseRequest, mapping, context, false) HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/" + scopeIndex, newIndexSettings, null) } diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEventSession.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEventSession.groovy index a04a4b097..c692fa7a2 100644 --- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEventSession.groovy +++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEventSession.groovy @@ -58,7 +58,7 @@ Collections.sort(eventSortedIndices) context.performMigrationStep("2.2.0-migrate-existing-events", () -> { MigrationUtils.cleanAllIndexWithRollover(context.getHttpClient(), bundleContext, esAddress, indexPrefix, "event") eventSortedIndices.each { eventIndex -> - MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext, esAddress, eventIndex, indexPrefix + "-event") + MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext, esAddress, eventIndex, indexPrefix + "-event", null) sleep(3000) } }) @@ -86,7 +86,7 @@ Collections.sort(sessionSortedIndices) context.performMigrationStep("2.2.0-migrate-existing-sessions", () -> { MigrationUtils.cleanAllIndexWithRollover(context.getHttpClient(), bundleContext, esAddress, indexPrefix, "session") sessionSortedIndices.each { sessionIndex -> - MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext, esAddress, sessionIndex, indexPrefix + "-session") + MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext, esAddress, sessionIndex, indexPrefix + "-session", null) sleep(3000) } }) diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-05-indicesReduction.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-05-indicesReduction.groovy new file mode 100644 index 000000000..8a07a5278 --- /dev/null +++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-05-indicesReduction.groovy @@ -0,0 +1,78 @@ +import org.apache.unomi.shell.migration.service.MigrationContext +import org.apache.unomi.shell.migration.utils.HttpUtils +import org.apache.unomi.shell.migration.utils.MigrationUtils + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +MigrationContext context = migrationContext +String esAddress = context.getConfigString("esAddress") +String indexPrefix = context.getConfigString("indexPrefix") +String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json") + +context.performMigrationStep("2.2.0-create-systemItems-index", () -> { + if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, "${indexPrefix}-systemitems")) { + String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "systemItems.json") + String newIndexSettings = MigrationUtils.buildIndexCreationRequest(baseSettings, mapping, context, false) + HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/${indexPrefix}-systemitems", newIndexSettings, null) + } +}) + +def indicesToReduce = [ + actiontype: "systemitems", + campaign: "systemitems", + campaignevent: "systemitems", + goal: "systemitems", + userlist: "systemitems", + propertytype: "systemitems", + scope: "systemitems", + conditiontype: "systemitems", + rule: "systemitems", + scoring: "systemitems", + segment: "systemitems", + topic: "systemitems", + patch: "systemitems", + jsonschema: "systemitems", + importconfig: "systemitems", + exportconfig: "systemitems", + rulestats: "systemitems", + groovyaction: "systemitems", + persona: "profile", + personasession: "session" +] +def indicesToSuffixIds = [ + rulestats: "-stat", + groovyaction: "-groovySourceCode" +] +indicesToReduce.each { indexToReduce -> + context.performMigrationStep("2.2.0-reduce-${indexToReduce.key}", () -> { + if (MigrationUtils.indexExists(context.getHttpClient(), esAddress, "${indexPrefix}-${indexToReduce.key}")) { + def painless = null + // check if we need to update the ids of those items first + if (indicesToSuffixIds.containsKey(indexToReduce.key)) { + painless = MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.2.0/suffix_ids.painless").replace("#ID_SUFFIX", indicesToSuffixIds.get(indexToReduce.key)) + } + // move items + MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext, esAddress, "${indexPrefix}-${indexToReduce.key}", "${indexPrefix}-${indexToReduce.value}", painless) + MigrationUtils.deleteIndex(context.getHttpClient(), esAddress, "${indexPrefix}-${indexToReduce.key}") + HttpUtils.executePostRequest(context.getHttpClient(), esAddress + "/${indexPrefix}-${indexToReduce.value}/_refresh", null, null); + MigrationUtils.waitForYellowStatus(context.getHttpClient(), esAddress, context); + } + }) +} + + diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/campaign.json similarity index 100% rename from persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json rename to tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/campaign.json diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/conditionType.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/conditionType.json similarity index 100% rename from persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/conditionType.json rename to tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/conditionType.json diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/goal.json similarity index 100% rename from persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json rename to tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/goal.json diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/patch.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/patch.json similarity index 100% rename from persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/patch.json rename to tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/patch.json diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/rule.json similarity index 100% rename from persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json rename to tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/rule.json diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scope.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/scope.json similarity index 100% rename from persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scope.json rename to tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/scope.json diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/scoring.json similarity index 100% rename from persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json rename to tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/scoring.json diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/segment.json similarity index 100% rename from persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json rename to tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/segment.json diff --git a/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_reindex_request.json b/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_reindex_request.json index ddcb79a5e..589e71084 100644 --- a/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_reindex_request.json +++ b/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_reindex_request.json @@ -4,5 +4,5 @@ }, "dest": { "index": "#dest" - } + }#painless } diff --git a/tools/shell-commands/src/main/resources/requestBody/2.2.0/suffix_ids.painless b/tools/shell-commands/src/main/resources/requestBody/2.2.0/suffix_ids.painless new file mode 100644 index 000000000..3b63549f6 --- /dev/null +++ b/tools/shell-commands/src/main/resources/requestBody/2.2.0/suffix_ids.painless @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +ctx._id = ctx._id + '#ID_SUFFIX'; +ctx._source.itemId = ctx._id; \ No newline at end of file