This is an automated email from the ASF dual-hosted git repository.
jsinovassinnaik pushed a commit to branch UNOMI-775-add-validation-endpoint
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to
refs/heads/UNOMI-775-add-validation-endpoint by this push:
new 8a09fbb60 UNOMI-775 : add json schema validation endpoint for event
list
8a09fbb60 is described below
commit 8a09fbb600dfa6f51772cb3c8724f04ae67e6a5e
Author: jsinovassin <[email protected]>
AuthorDate: Fri Apr 28 18:40:39 2023 +0200
UNOMI-775 : add json schema validation endpoint for event list
---
.../unomi/schema/rest/JsonSchemaEndPoint.java | 27 +++++++--
.../org/apache/unomi/schema/api/SchemaService.java | 10 ++++
.../unomi/schema/impl/SchemaServiceImpl.java | 58 +++++++++++++++++---
.../java/org/apache/unomi/itests/JSONSchemaIT.java | 64 +++++++++++++++++++---
4 files changed, 139 insertions(+), 20 deletions(-)
diff --git
a/extensions/json-schema/rest/src/main/java/org/apache/unomi/schema/rest/JsonSchemaEndPoint.java
b/extensions/json-schema/rest/src/main/java/org/apache/unomi/schema/rest/JsonSchemaEndPoint.java
index fdb919538..c157370fd 100644
---
a/extensions/json-schema/rest/src/main/java/org/apache/unomi/schema/rest/JsonSchemaEndPoint.java
+++
b/extensions/json-schema/rest/src/main/java/org/apache/unomi/schema/rest/JsonSchemaEndPoint.java
@@ -36,8 +36,7 @@ import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
-import java.util.Collection;
-import java.util.Set;
+import java.util.*;
@WebService
@Produces(MediaType.APPLICATION_JSON + ";charset=UTF-8")
@@ -95,7 +94,7 @@ public class JsonSchemaEndPoint {
*/
@POST
@Path("/")
- @Consumes({ MediaType.TEXT_PLAIN, MediaType.APPLICATION_JSON })
+ @Consumes({MediaType.TEXT_PLAIN, MediaType.APPLICATION_JSON})
@Produces(MediaType.APPLICATION_JSON)
public Response save(String jsonSchema) {
try {
@@ -119,12 +118,13 @@ public class JsonSchemaEndPoint {
/**
* Being able to validate a given event is useful when you want to develop
custom events and associated schemas
+ *
* @param event the event to be validated
* @return Validation error messages if there is some
*/
@POST
@Produces(MediaType.APPLICATION_JSON + ";charset=UTF-8")
- @Consumes({ MediaType.TEXT_PLAIN, MediaType.APPLICATION_JSON })
+ @Consumes({MediaType.TEXT_PLAIN, MediaType.APPLICATION_JSON})
@Path("/validateEvent")
public Collection<ValidationError> validateEvent(String event) {
try {
@@ -134,4 +134,23 @@ public class JsonSchemaEndPoint {
throw new InvalidRequestException(errorMessage, errorMessage);
}
}
+
+ /**
+ * Being able to validate a given list of event is useful when you want to
develop custom events and associated schemas
+ *
+ * @param events the events to be validated
+ * @return Validation error messages if there is some grouped per event
type
+ */
+ @POST
+ @Produces(MediaType.APPLICATION_JSON + ";charset=UTF-8")
+ @Consumes({MediaType.TEXT_PLAIN, MediaType.APPLICATION_JSON})
+ @Path("/validateEvents")
+ public Map<String, Set<ValidationError>> validateEvents(String events) {
+ try {
+ return schemaService.validateEvents(events);
+ } catch (Exception e) {
+ String errorMessage = "Unable to validate event: " +
e.getMessage();
+ throw new InvalidRequestException(errorMessage, errorMessage);
+ }
+ }
}
diff --git
a/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/api/SchemaService.java
b/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/api/SchemaService.java
index 2690f5ba8..162ecbe77 100644
---
a/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/api/SchemaService.java
+++
b/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/api/SchemaService.java
@@ -20,6 +20,7 @@ package org.apache.unomi.schema.api;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
+import java.util.Map;
import java.util.Set;
/**
@@ -63,6 +64,15 @@ public interface SchemaService {
*/
Set<ValidationError> validateEvent(String event) throws
ValidationException;
+ /**
+ * perform a validation of a list of the given events
+ *
+ * @param events the events to validate
+ * @return The Map of validation errors group per event type in case there
is some, empty map otherwise
+ * @throws ValidationException in case something goes wrong and validation
could not be performed.
+ */
+ Map<String,Set<ValidationError>> validateEvents(String events) throws
ValidationException;
+
/**
* Get the list of installed Json Schema Ids
*
diff --git
a/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/impl/SchemaServiceImpl.java
b/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/impl/SchemaServiceImpl.java
index 2371738b6..7da059fa6 100644
---
a/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/impl/SchemaServiceImpl.java
+++
b/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/impl/SchemaServiceImpl.java
@@ -40,6 +40,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
+import java.text.MessageFormat;
import java.util.*;
import java.util.concurrent.*;
import java.util.stream.Collectors;
@@ -54,7 +55,7 @@ public class SchemaServiceImpl implements SchemaService {
ObjectMapper objectMapper = new ObjectMapper();
/**
- * Schemas provided by Unomi runtime bundles in /META-INF/cxs/schemas/...
+ * Schemas provided by Unomi runtime bundles in /META-INF/cxs/schemas/...
*/
private final ConcurrentMap<String, JsonSchemaWrapper>
predefinedUnomiJSONSchemaById = new ConcurrentHashMap<>();
/**
@@ -111,12 +112,52 @@ public class SchemaServiceImpl implements SchemaService {
@Override
public Set<ValidationError> validateEvent(String event) throws
ValidationException {
- JsonNode jsonEvent = parseData(event);
- String eventType = extractEventType(jsonEvent);
+ return validateNodeEvent(parseData(event));
+ }
+
+ @Override
+ public Map<String, Set<ValidationError>> validateEvents(String events)
throws ValidationException {
+ Map<String, Set<ValidationError>> errorsPerEventType = new HashMap<>();
+ JsonNode eventsNodes = parseData(events);
+ eventsNodes.forEach(event -> {
+ String eventType = event.get("eventType").asText();
+ try {
+ Set<ValidationError> errors = validateNodeEvent(event);
+ if (errorsPerEventType.containsKey(eventType)) {
+ errorsPerEventType.get(eventType).addAll(errors);
+ } else {
+ errorsPerEventType.put(eventType, errors);
+ }
+ } catch (ValidationException e) {
+ Set<ValidationError> errors = buildCustomErrorMessage();
+ if (errorsPerEventType.containsKey(eventType)) {
+ errorsPerEventType.get(eventType).addAll(errors);
+ } else {
+ errorsPerEventType.put(eventType, errors);
+ }
+ errorsPerEventType.put(eventType, errors);
+
+ logger.debug(e.getMessage());
+ }
+ });
+ return errorsPerEventType;
+ }
+
+ private Set<ValidationError> buildCustomErrorMessage() {
+ ValidationMessage.Builder builder = new ValidationMessage.Builder();
+ builder.customMessage("No Schema found for this event
type").format(new MessageFormat("Not used pattern. Message format is
required"));
+ ValidationError error = new ValidationError(builder.build());
+ Set<ValidationError> errors = new HashSet<>();
+ errors.add(error);
+ return errors;
+ }
+
+ private Set<ValidationError> validateNodeEvent(JsonNode event) throws
ValidationException {
+ String eventType = extractEventType(event);
JsonSchemaWrapper eventSchema = getSchemaForEventType(eventType);
JsonSchema jsonSchema = getJsonSchema(eventSchema.getItemId());
- return validate(jsonEvent, jsonSchema);
+ return validate(event, jsonSchema);
}
@Override
@@ -145,9 +186,9 @@ public class SchemaServiceImpl implements SchemaService {
return schemasById.values().stream()
.filter(jsonSchemaWrapper ->
jsonSchemaWrapper.getTarget() != null &&
- jsonSchemaWrapper.getTarget().equals(TARGET_EVENTS) &&
- jsonSchemaWrapper.getName() != null &&
- jsonSchemaWrapper.getName().equals(eventType))
+
jsonSchemaWrapper.getTarget().equals(TARGET_EVENTS) &&
+ jsonSchemaWrapper.getName() != null &&
+ jsonSchemaWrapper.getName().equals(eventType))
.findFirst()
.orElseThrow(() -> new ValidationException("Schema not found
for event type: " + eventType));
}
@@ -211,7 +252,6 @@ public class SchemaServiceImpl implements SchemaService {
if (StringUtils.isEmpty(data)) {
throw new ValidationException("Empty data, nothing to validate");
}
-
try {
return objectMapper.readTree(data);
} catch (Exception e) {
@@ -318,7 +358,7 @@ public class SchemaServiceImpl implements SchemaService {
ArrayNode allOf;
if (jsonSchema.at("/allOf") instanceof MissingNode) {
allOf = objectMapper.createArrayNode();
- } else if (jsonSchema.at("/allOf") instanceof ArrayNode){
+ } else if (jsonSchema.at("/allOf") instanceof ArrayNode) {
allOf = (ArrayNode) jsonSchema.at("/allOf");
} else {
logger.warn("Cannot extends schema allOf property, it should
be an Array, please fix your schema definition for schema: {}", id);
diff --git a/itests/src/test/java/org/apache/unomi/itests/JSONSchemaIT.java
b/itests/src/test/java/org/apache/unomi/itests/JSONSchemaIT.java
index 43cb5949d..f3b24a936 100644
--- a/itests/src/test/java/org/apache/unomi/itests/JSONSchemaIT.java
+++ b/itests/src/test/java/org/apache/unomi/itests/JSONSchemaIT.java
@@ -30,6 +30,7 @@ import org.apache.unomi.api.services.ScopeService;
import org.apache.unomi.itests.tools.httpclient.HttpClientThatWaitsForUnomi;
import org.apache.unomi.schema.api.JsonSchemaWrapper;
import org.apache.unomi.schema.api.SchemaService;
+import org.apache.unomi.schema.api.ValidationError;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -47,6 +48,8 @@ import java.util.*;
import static org.junit.Assert.*;
+import java.util.stream.Collectors;
+
/**
* Class to tests the JSON schema features
*/
@@ -67,7 +70,7 @@ public class JSONSchemaIT extends BaseIT {
DEFAULT_TRYING_TRIES);
TestUtils.createScope(DUMMY_SCOPE, "Dummy scope", scopeService);
- keepTrying("Scope "+ DUMMY_SCOPE +" not found in the required time",
() -> scopeService.getScope(DUMMY_SCOPE),
+ keepTrying("Scope " + DUMMY_SCOPE + " not found in the required time",
() -> scopeService.getScope(DUMMY_SCOPE),
Objects::nonNull, DEFAULT_TRYING_TIMEOUT,
DEFAULT_TRYING_TRIES);
}
@@ -242,6 +245,53 @@ public class JSONSchemaIT extends BaseIT {
DEFAULT_TRYING_TRIES);
}
+ @Test
+ public void testValidateEvents_valid() throws Exception {
+
assertNull(schemaService.getSchema("https://vendor.test.com/schemas/json/events/flattened/1-0-0";));
+
assertNull(schemaService.getSchema("https://vendor.test.com/schemas/json/events/flattened/properties/1-0-0";));
+
assertNull(schemaService.getSchema("https://vendor.test.com/schemas/json/events/flattened/properties/interests/1-0-0";));
+
+ // Test that at first the flattened event is not valid.
+
assertFalse(schemaService.isEventValid(resourceAsString("schemas/event-flattened-valid.json")));
+
+ // save schemas and check the event pass the validation
+
schemaService.saveSchema(resourceAsString("schemas/schema-flattened.json"));
+
schemaService.saveSchema(resourceAsString("schemas/schema-flattened-flattenedProperties.json"));
+
schemaService.saveSchema(resourceAsString("schemas/schema-flattened-flattenedProperties-interests.json"));
+
schemaService.saveSchema(resourceAsString("schemas/schema-flattened-properties.json"));
+
+ StringBuilder listEvents = new StringBuilder("");
+ listEvents
+ .append("[")
+ .append(resourceAsString("schemas/event-flattened-valid.json"))
+ .append("]");
+
+ keepTrying("No error should have been detected",
+ () -> {
+ try {
+ return
schemaService.validateEvents(listEvents.toString()).isEmpty();
+ } catch (Exception e) {
+ return false;
+ }
+ },
+ isValid -> isValid, DEFAULT_TRYING_TIMEOUT,
DEFAULT_TRYING_TRIES);
+
+ StringBuilder listInvalidEvents = new StringBuilder("");
+ listInvalidEvents
+ .append("[")
+
.append(resourceAsString("schemas/event-flattened-invalid-1.json")).append(",")
+
.append(resourceAsString("schemas/event-flattened-invalid-2.json")).append(",")
+
.append(resourceAsString("schemas/event-flattened-invalid-3.json")).append(",")
+
.append(resourceAsString("schemas/event-flattened-invalid-3.json"))
+ .append("]");
+ Map<String, Set<ValidationError>> errors =
schemaService.validateEvents(listInvalidEvents.toString());
+
+ assertEquals(9, errors.size());
+ // Verify that error on interests.football appear only once even if
two events have the issue
+ assertEquals(1,
errors.get("flattened").stream().filter(validationError ->
validationError.getError().startsWith("$.flattenedProperties.interests.football")).collect(Collectors.toList()).size());
+ }
+
+
@Test
public void testFlattenedProperties() throws Exception {
assertNull(schemaService.getSchema("https://vendor.test.com/schemas/json/events/flattened/1-0-0";));
@@ -269,15 +319,15 @@ public class JSONSchemaIT extends BaseIT {
// check that range query is not working on flattened props:
Condition condition = new
Condition(definitionsService.getConditionType("eventPropertyCondition"));
-
condition.setParameter("propertyName","flattenedProperties.interests.cars");
- condition.setParameter("comparisonOperator","greaterThan");
+ condition.setParameter("propertyName",
"flattenedProperties.interests.cars");
+ condition.setParameter("comparisonOperator", "greaterThan");
condition.setParameter("propertyValueInteger", 2);
assertNull(persistenceService.query(condition, null, Event.class, 0,
-1));
// check that term query is working on flattened props:
condition = new
Condition(definitionsService.getConditionType("eventPropertyCondition"));
-
condition.setParameter("propertyName","flattenedProperties.interests.cars");
- condition.setParameter("comparisonOperator","equals");
+ condition.setParameter("propertyName",
"flattenedProperties.interests.cars");
+ condition.setParameter("comparisonOperator", "equals");
condition.setParameter("propertyValueInteger", 15);
List<Event> events = persistenceService.query(condition, null,
Event.class, 0, -1).getList();
assertEquals(1, events.size());
@@ -325,8 +375,8 @@ public class JSONSchemaIT extends BaseIT {
// wait for the event to be indexed
Condition condition = new
Condition(definitionsService.getConditionType("eventPropertyCondition"));
- condition.setParameter("propertyName","properties.marker.keyword");
- condition.setParameter("comparisonOperator","equals");
+ condition.setParameter("propertyName", "properties.marker.keyword");
+ condition.setParameter("comparisonOperator", "equals");
condition.setParameter("propertyValue", eventMarker);
List<Event> events = keepTrying("The event should have been persisted",
() -> persistenceService.query(condition, null, Event.class),
results -> results.size() == 1,
