This is an automated email from the ASF dual-hosted git repository.
dgriffon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/master by this push:
new 56e896f20 UNOMI-816 : ensure empty scope are filled properly at
migration time (#666)
56e896f20 is described below
commit 56e896f20cf81163a523d157d55af1deebc304e4
Author: David Griffon <dgrif...@jahia.com>
AuthorDate: Fri Jun 7 09:51:14 2024 +0200
UNOMI-816 : ensure empty scope are filled properly at migration time (#666)
* UNOMI-816 : ensure empty scope are filled properly at migration time
* UNOMI-816 : fix archive to match ES version (+ improve doc)
* UNOMI-816 : fix test
* UNOMI-816 : move scope migration execution to 2.5.0
* UNOMI-816 : small improvements
---
itests/README.md | 17 ++++
.../unomi/itests/migration/Migrate16xTo220IT.java | 90 ++++++++++++++-------
.../migration/match_all_login_event_request.json | 10 +++
.../resources/migration/snapshots_repository.zip | Bin 7807889 -> 3872664
bytes
.../migrate-2.4.0-15-viewEventPagePath.groovy | 5 +-
...ovy => migrate-2.5.0-10-loginEventScope.groovy} | 11 ++-
.../2.5.0/login_event_scope_migrate.painless | 26 ++++++
.../2.5.0/scope_update_by_query_request.json | 29 +++++++
8 files changed, 155 insertions(+), 33 deletions(-)
diff --git a/itests/README.md b/itests/README.md
index 7be12dcd0..b49831889 100644
--- a/itests/README.md
+++ b/itests/README.md
@@ -172,14 +172,27 @@ This snapshot has been done on Unomi 1.6.x using
ElasticSearch 7.11.0.
So we will set up locally those servers in the exact same versions.
(For now just download them and do not start them yet.)
+To ease the migration, you can run the docker image of ElasticSearch 7.11.0
with the following command:
+
+ docker run -p 9200:9200 -e path.repo="/tmp/snapshots_repository" -e
discovery.type=single-node docker.elastic.co/elasticsearch/elasticsearch:7.11.0
+
+Note that the path.repo is set to `/tmp/snapshots_repository` so you can use
this path to store the snapshot repository.
+
First we need to extract the zip of the snapshot repository from the test
resources:
/src/test/resources/migration/snapshots_repository.zip
+
+> If you use docker, you can copy the zip file to the docker container
using the following command:
+> docker cp src/test/resources/migration/snapshots_repository.zip
<container_id>:/tmp/snapshots_repository.zip
+
+Then unzip it to the path you want to use as snapshot repository.
In my case I unzip it to:
/servers/elasticsearch-7.11.0/
+> For docker unzip in the `/tmp` folder.
+
So I have the following folders structure:
/servers/elasticsearch-7.11.0/snapshots_repository/snapshots
@@ -190,6 +203,8 @@ Now we need to configure our ElasticSearch server to allow
this path as repo, ed
repo:
- /servers/elasticsearch-7.11.0/snapshots_repository
+> This step is not required for docker.
+
Start ElasticSearch server.
Now we have to add the snapshot repository, do the following request on your
ElasticSearch instance:
@@ -242,4 +257,6 @@ And the final step is, zipping the new version of the
snapshot repository and re
zip -r snapshots_repository.zip
/servers/elasticsearch-7.11.0/snapshots_repository
cp /servers/elasticsearch-7.11.0/snapshots_repository.zip
src/test/resources/migration/snapshots_repository.zip
+> In case you are using docker, do zip in the container and use `docker cp` to
get the zip file from the docker container.
+
Now you can modify the migration test class to test that your added data in
1.6.x is correctly migrated in 2.0.0
diff --git
a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java
b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java
index 725f0fa4c..f892c338a 100644
---
a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java
+++
b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java
@@ -29,16 +29,15 @@ import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
+import java.util.*;
public class Migrate16xTo220IT extends BaseIT {
private int eventCount = 0;
private int sessionCount = 0;
+ private Set<String[]> initialScopes = new HashSet<>();
+ private static final String SCOPE_NOT_EXIST = "SCOPE_NOT_EXIST";
private static final int NUMBER_DUPLICATE_SESSIONS = 3;
private static final List<String> oldSystemItemsIndices =
Arrays.asList("context-actiontype", "context-campaign",
"context-campaignevent", "context-goal",
"context-userlist", "context-propertytype", "context-scope",
"context-conditiontype", "context-rule", "context-scoring", "context-segment",
"context-groovyaction", "context-topic",
@@ -100,6 +99,7 @@ public class Migrate16xTo220IT extends BaseIT {
checkIndexReductions2_2_0();
checkPagePathForEventView();
checkPastEvents();
+ checkScopeEventHaveBeenUpdated();
}
/**
@@ -269,6 +269,17 @@ public class Migrate16xTo220IT extends BaseIT {
}
}
+ private void checkScopeEventHaveBeenUpdated() {
+ for (String[] loginEvent : initialScopes) {
+ Event event = eventService.getEvent(loginEvent[0]);
+ if ("digitall".equals(loginEvent[1])) {
+ Assert.assertEquals(event.getScope(), "digitall");
+ } else {
+ Assert.assertEquals(event.getScope(), "systemsite");
+ }
+ }
+ }
+
/**
* Data set contains a profile (id: e67ecc69-a7b3-47f1-b91f-5d6e7b90276e)
with two interests: football:50 and basketball:30
* Also it's first name is test_profile
@@ -316,6 +327,7 @@ public class Migrate16xTo220IT extends BaseIT {
private void initCounts(CloseableHttpClient httpClient) {
try {
for (String eventIndex :
MigrationUtils.getIndexesPrefixedBy(httpClient, "http://localhost:9400";,
"context-event-date")) {
+ getScopeFromEvents(httpClient, eventIndex);
eventCount += countItems(httpClient, eventIndex,
resourceAsString("migration/must_not_match_some_eventype_body.json"));
}
@@ -327,33 +339,55 @@ public class Migrate16xTo220IT extends BaseIT {
}
}
- private int countItems(CloseableHttpClient httpClient, String index,
String requestBody) throws IOException {
- if (requestBody == null) {
- requestBody =
resourceAsString("migration/must_not_match_some_eventype_body.json");
+ private void getScopeFromEvents(CloseableHttpClient httpClient, String
eventIndex) throws IOException {
+ String requestBody =
resourceAsString("migration/match_all_login_event_request.json");
+ JsonNode jsonNode =
objectMapper.readTree(HttpUtils.executePostRequest(httpClient,
"http://localhost:9400"; + "/" + eventIndex + "/_search", requestBody, null));
+ if (jsonNode.has("hits") && jsonNode.get("hits").has("hits") &&
!jsonNode.get("hits").get("hits").isEmpty()) {
+ jsonNode.get("hits").get("hits").forEach(doc -> {
+ JsonNode event = doc.get("_source");
+ if (event.has("scope")) {
+ if (event.get("scope") == null) {
+ String[] initialScope = {event.get("itemId").asText(),
null};
+ initialScopes.add(initialScope);
+ } else {
+ String[] initialScope = {event.get("itemId").asText(),
event.get("scope").asText()};
+ initialScopes.add(initialScope);
+ }
+ } else {
+ String[] initialScope = {event.get("itemId").asText(),
SCOPE_NOT_EXIST};
+ initialScopes.add(initialScope);
+ }
+ });
}
- JsonNode jsonNode =
objectMapper.readTree(HttpUtils.executePostRequest(httpClient,
"http://localhost:9400"; + "/" + index + "/_count", requestBody, null));
- return jsonNode.get("count").asInt();
}
- /**
- * Data set contains 2 events that had a value in properties.path:
- * The properties.path should have been moved to
properties.pageInfo.pagePath
- */
- private void checkPagePathForEventView() {
- Assert.assertEquals(2,
persistenceService.query("target.properties.pageInfo.pagePath",
"/path/to/migrate/to/pageInfo", null, Event.class).size());
- Assert.assertEquals(0, persistenceService.query("properties.path",
"/path/to/migrate/to/pageInfo", null, Event.class).size());
- }
+ private int countItems (CloseableHttpClient httpClient, String index,
String requestBody) throws IOException {
+ if (requestBody == null) {
+ requestBody =
resourceAsString("migration/must_not_match_some_eventype_body.json");
+ }
+ JsonNode jsonNode =
objectMapper.readTree(HttpUtils.executePostRequest(httpClient,
"http://localhost:9400"; + "/" + index + "/_count", requestBody, null));
+ return jsonNode.get("count").asInt();
+ }
+ /**
+ * Data set contains 2 events that had a value in properties.path:
+ * The properties.path should have been moved to
properties.pageInfo.pagePath
+ */
+ private void checkPagePathForEventView () {
+ Assert.assertEquals(2,
persistenceService.query("target.properties.pageInfo.pagePath",
"/path/to/migrate/to/pageInfo", null, Event.class).size());
+ Assert.assertEquals(0, persistenceService.query("properties.path",
"/path/to/migrate/to/pageInfo", null, Event.class).size());
+ }
- /**
- * Data set contains a profile (id: 164adad8-6885-45b6-8e9d-512bf4a7d10d)
with a system property pastEvents that contains 5 events with key
eventTriggeredabcdefgh
- * This test ensures that the pastEvents have been migrated to the new
data structure
- */
- private void checkPastEvents() {
- Profile profile =
persistenceService.load("164adad8-6885-45b6-8e9d-512bf4a7d10d", Profile.class);
- List<Map<String, Object>> pastEvents = ((List<Map<String,
Object>>)profile.getSystemProperties().get("pastEvents"));
- Assert.assertEquals(1, pastEvents.size());
- Assert.assertEquals("eventTriggeredabcdefgh",
pastEvents.get(0).get("key"));
- Assert.assertEquals(5, (int) pastEvents.get(0).get("count"));
+
+ /**
+ * Data set contains a profile (id:
164adad8-6885-45b6-8e9d-512bf4a7d10d) with a system property pastEvents that
contains 5 events with key eventTriggeredabcdefgh
+ * This test ensures that the pastEvents have been migrated to the new
data structure
+ */
+ private void checkPastEvents () {
+ Profile profile =
persistenceService.load("164adad8-6885-45b6-8e9d-512bf4a7d10d", Profile.class);
+ List<Map<String, Object>> pastEvents = ((List<Map<String,
Object>>) profile.getSystemProperties().get("pastEvents"));
+ Assert.assertEquals(1, pastEvents.size());
+ Assert.assertEquals("eventTriggeredabcdefgh",
pastEvents.get(0).get("key"));
+ Assert.assertEquals(5, (int) pastEvents.get(0).get("count"));
+ }
}
-}
diff --git
a/itests/src/test/resources/migration/match_all_login_event_request.json
b/itests/src/test/resources/migration/match_all_login_event_request.json
new file mode 100644
index 000000000..24fb0bb38
--- /dev/null
+++ b/itests/src/test/resources/migration/match_all_login_event_request.json
@@ -0,0 +1,10 @@
+{
+ "query": {
+ "term": {
+ "eventType": {
+ "value": "login",
+ "boost": 1.0
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/itests/src/test/resources/migration/snapshots_repository.zip
b/itests/src/test/resources/migration/snapshots_repository.zip
index 675e710cc..566a5657c 100644
Binary files a/itests/src/test/resources/migration/snapshots_repository.zip and
b/itests/src/test/resources/migration/snapshots_repository.zip differ
diff --git
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.4.0-15-viewEventPagePath.groovy
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.4.0-15-viewEventPagePath.groovy
index faaac944d..8eb0de38d 100644
---
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.4.0-15-viewEventPagePath.groovy
+++
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.4.0-15-viewEventPagePath.groovy
@@ -24,7 +24,10 @@ String esAddress = context.getConfigString("esAddress")
String indexPrefix = context.getConfigString("indexPrefix")
context.performMigrationStep("2.4.0-migrate-view-event-page-path", () -> {
+ Set<String> eventIndices =
MigrationUtils.getIndexesPrefixedBy(context.getHttpClient(), esAddress,
"${indexPrefix}-event")
String updatePathScript =
MigrationUtils.getFileWithoutComments(bundleContext,
"requestBody/2.4.0/view_event_page_path_migrate.painless")
String baseSettings = MigrationUtils.resourceAsString(bundleContext,
"requestBody/2.4.0/base_update_by_query_request.json")
- HttpUtils.executePostRequest(context.getHttpClient(),
"${esAddress}/${indexPrefix}-event/_update_by_query",
baseSettings.replace('#painless', updatePathScript), null)
+ eventIndices.each { eventIndex ->
+ HttpUtils.executePostRequest(context.getHttpClient(),
"${esAddress}/${eventIndex}/_update_by_query",
baseSettings.replace('#painless', updatePathScript), null)
+ }
})
diff --git
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.4.0-15-viewEventPagePath.groovy
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.5.0-10-loginEventScope.groovy
similarity index 69%
copy from
tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.4.0-15-viewEventPagePath.groovy
copy to
tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.5.0-10-loginEventScope.groovy
index faaac944d..24e2042e6 100644
---
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.4.0-15-viewEventPagePath.groovy
+++
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.5.0-10-loginEventScope.groovy
@@ -23,8 +23,11 @@ MigrationContext context = migrationContext
String esAddress = context.getConfigString("esAddress")
String indexPrefix = context.getConfigString("indexPrefix")
-context.performMigrationStep("2.4.0-migrate-view-event-page-path", () -> {
- String updatePathScript =
MigrationUtils.getFileWithoutComments(bundleContext,
"requestBody/2.4.0/view_event_page_path_migrate.painless")
- String baseSettings = MigrationUtils.resourceAsString(bundleContext,
"requestBody/2.4.0/base_update_by_query_request.json")
- HttpUtils.executePostRequest(context.getHttpClient(),
"${esAddress}/${indexPrefix}-event/_update_by_query",
baseSettings.replace('#painless', updatePathScript), null)
+context.performMigrationStep("2.5.0-migrate-login-event-scope", () -> {
+ Set<String> eventIndices =
MigrationUtils.getIndexesPrefixedBy(context.getHttpClient(), esAddress,
"${indexPrefix}-event")
+ String updatePathScript =
MigrationUtils.getFileWithoutComments(bundleContext,
"requestBody/2.5.0/login_event_scope_migrate.painless")
+ String baseSettings = MigrationUtils.resourceAsString(bundleContext,
"requestBody/2.5.0/scope_update_by_query_request.json")
+ eventIndices.each { eventIndex ->
+ HttpUtils.executePostRequest(context.getHttpClient(),
"${esAddress}/${eventIndex}/_update_by_query",
baseSettings.replace('#painless', updatePathScript), null)
+ }
})
diff --git
a/tools/shell-commands/src/main/resources/requestBody/2.5.0/login_event_scope_migrate.painless
b/tools/shell-commands/src/main/resources/requestBody/2.5.0/login_event_scope_migrate.painless
new file mode 100644
index 000000000..837103fe1
--- /dev/null
+++
b/tools/shell-commands/src/main/resources/requestBody/2.5.0/login_event_scope_migrate.painless
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Handle login events */
+
+ctx._source.put('scope', 'systemsite');
+if (ctx._source.source != null) {
+ ctx._source.source.put('scope', 'systemsite');
+}
+if (ctx._source.target != null) {
+ ctx._source.target.put('scope', 'systemsite');
+}
diff --git
a/tools/shell-commands/src/main/resources/requestBody/2.5.0/scope_update_by_query_request.json
b/tools/shell-commands/src/main/resources/requestBody/2.5.0/scope_update_by_query_request.json
new file mode 100644
index 000000000..7b2a7b54b
--- /dev/null
+++
b/tools/shell-commands/src/main/resources/requestBody/2.5.0/scope_update_by_query_request.json
@@ -0,0 +1,29 @@
+{
+ "script": {
+ "source": "#painless",
+ "lang": "painless"
+ },
+ "query": {
+ "bool": {
+ "must": [
+ {
+ "match": {
+ "itemType": "event"
+ }
+ },
+ {
+ "match": {
+ "eventType": "login"
+ }
+ }
+ ],
+ "must_not": [
+ {
+ "exists": {
+ "field": "scope"
+ }
+ }
+ ]
+ }
+ }
+}
