This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch
3282-dd-regex-based-preprocessing-rule-to-adapters
in repository https://gitbox.apache.org/repos/asf/streampipes.git
The following commit(s) were added to
refs/heads/3282-dd-regex-based-preprocessing-rule-to-adapters by this push:
new bbc0a69844 feat(#3282): Add a regex rule for string event properties
to connect
bbc0a69844 is described below
commit bbc0a69844b0260a5010d431d30b6edb2f4bc5be
Author: Philipp Zehnder <[email protected]>
AuthorDate: Mon Oct 7 15:43:28 2024 +0200
feat(#3282): Add a regex rule for string event properties to connect
---
.../connect/TransformationRuleUpdateVisitor.java | 8 ++
.../rules/TransformationRuleDescription.java | 2 +
.../storage/couchdb/serializer/GsonSerializer.java | 3 +
.../connect/regexRule/expectedReplaceAllFalse.csv | 2 +
.../connect/regexRule/expectedReplaceAllTrue.csv | 2 +
.../expectedReplaceAllTrueWithEmptyString.csv | 2 +
ui/cypress/fixtures/connect/regexRule/input.csv | 2 +
.../utils/connect/ConnectEventSchemaUtils.ts | 17 +++-
.../connect/rules/deleteTransformationRule.spec.ts | 12 +--
ui/cypress/tests/connect/rules/regexRule.spec.ts | 107 +++++++++++++++++++++
.../src/lib/model/gen/streampipes-model-client.ts | 5 +-
.../src/lib/model/gen/streampipes-model.ts | 37 ++++++-
.../src/lib/model/types/data-type.ts | 4 +
ui/src/app/connect/connect.module.ts | 2 +
.../edit-regex-transformation.component.html | 52 ++++++++++
.../edit-regex-transformation.component.scss | 17 ++++
.../edit-regex-transformation.component.ts | 29 ++++++
.../edit-value-transformation.component.html | 10 ++
.../edit-value-transformation.component.ts | 1 +
.../edit-event-property.component.html | 12 +--
.../edit-event-property.component.ts | 18 ++++
.../services/transformation-rule.service.ts | 63 ++++++++++++
22 files changed, 383 insertions(+), 24 deletions(-)
diff --git
a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/TransformationRuleUpdateVisitor.java
b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/TransformationRuleUpdateVisitor.java
index ef357e4c48..9661b7705c 100644
---
a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/TransformationRuleUpdateVisitor.java
+++
b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/TransformationRuleUpdateVisitor.java
@@ -31,6 +31,7 @@ import
org.apache.streampipes.model.connect.rules.value.AddTimestampRuleDescript
import
org.apache.streampipes.model.connect.rules.value.AddValueTransformationRuleDescription;
import
org.apache.streampipes.model.connect.rules.value.ChangeDatatypeTransformationRuleDescription;
import
org.apache.streampipes.model.connect.rules.value.CorrectionValueTransformationRuleDescription;
+import
org.apache.streampipes.model.connect.rules.value.RegexTransformationRuleDescription;
import
org.apache.streampipes.model.connect.rules.value.TimestampTranfsformationRuleDescription;
import
org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription;
import org.apache.streampipes.model.schema.EventProperty;
@@ -80,6 +81,13 @@ public class TransformationRuleUpdateVisitor implements
ITransformationRuleVisit
}
}
+ @Override
+ public void visit(RegexTransformationRuleDescription rule) {
+ if (containsKey(rule.getRuntimeKey())) {
+ validRules.add(rule);
+ }
+ }
+
@Override
public void visit(EventRateTransformationRuleDescription rule) {
// Do nothing
diff --git
a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/TransformationRuleDescription.java
b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/TransformationRuleDescription.java
index 0ea2194456..94b1817e8f 100644
---
a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/TransformationRuleDescription.java
+++
b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/TransformationRuleDescription.java
@@ -28,6 +28,7 @@ import
org.apache.streampipes.model.connect.rules.value.AddTimestampRuleDescript
import
org.apache.streampipes.model.connect.rules.value.AddValueTransformationRuleDescription;
import
org.apache.streampipes.model.connect.rules.value.ChangeDatatypeTransformationRuleDescription;
import
org.apache.streampipes.model.connect.rules.value.CorrectionValueTransformationRuleDescription;
+import
org.apache.streampipes.model.connect.rules.value.RegexTransformationRuleDescription;
import
org.apache.streampipes.model.connect.rules.value.TimestampTranfsformationRuleDescription;
import
org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription;
import org.apache.streampipes.model.shared.annotation.TsModel;
@@ -47,6 +48,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
@JsonSubTypes.Type(CreateNestedRuleDescription.class),
@JsonSubTypes.Type(DeleteRuleDescription.class),
@JsonSubTypes.Type(RenameRuleDescription.class),
+ @JsonSubTypes.Type(RegexTransformationRuleDescription.class),
@JsonSubTypes.Type(MoveRuleDescription.class),
@JsonSubTypes.Type(ChangeDatatypeTransformationRuleDescription.class),
@JsonSubTypes.Type(CorrectionValueTransformationRuleDescription.class),
diff --git
a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/serializer/GsonSerializer.java
b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/serializer/GsonSerializer.java
index 685db7ee03..fee35a57a3 100644
---
a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/serializer/GsonSerializer.java
+++
b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/serializer/GsonSerializer.java
@@ -35,6 +35,7 @@ import
org.apache.streampipes.model.connect.rules.value.AddTimestampRuleDescript
import
org.apache.streampipes.model.connect.rules.value.AddValueTransformationRuleDescription;
import
org.apache.streampipes.model.connect.rules.value.ChangeDatatypeTransformationRuleDescription;
import
org.apache.streampipes.model.connect.rules.value.CorrectionValueTransformationRuleDescription;
+import
org.apache.streampipes.model.connect.rules.value.RegexTransformationRuleDescription;
import
org.apache.streampipes.model.connect.rules.value.TimestampTranfsformationRuleDescription;
import
org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription;
import org.apache.streampipes.model.grounding.TopicDefinition;
@@ -94,6 +95,8 @@ public class GsonSerializer {
builder.registerTypeAdapterFactory(RuntimeTypeAdapterFactory.of(TransformationRuleDescription.class,
"sourceType")
.registerSubtype(RenameRuleDescription.class,
"org.apache.streampipes.model.RenameRuleDescription")
+ .registerSubtype(RegexTransformationRuleDescription.class,
+
"org.apache.streampipes.model.RegexTransformationRuleDescription")
.registerSubtype(MoveRuleDescription.class,
"org.apache.streampipes.model.MoveRuleDescription")
.registerSubtype(DeleteRuleDescription.class,
"org.apache.streampipes.model.DeleteRuleDescription")
.registerSubtype(CreateNestedRuleDescription.class,
"org.apache.streampipes.model.CreateNestedRuleDescription")
diff --git a/ui/cypress/fixtures/connect/regexRule/expectedReplaceAllFalse.csv
b/ui/cypress/fixtures/connect/regexRule/expectedReplaceAllFalse.csv
new file mode 100644
index 0000000000..f951ba2245
--- /dev/null
+++ b/ui/cypress/fixtures/connect/regexRule/expectedReplaceAllFalse.csv
@@ -0,0 +1,2 @@
+density;sensorId
+122.0;aensor01
diff --git a/ui/cypress/fixtures/connect/regexRule/expectedReplaceAllTrue.csv
b/ui/cypress/fixtures/connect/regexRule/expectedReplaceAllTrue.csv
new file mode 100644
index 0000000000..cbff946483
--- /dev/null
+++ b/ui/cypress/fixtures/connect/regexRule/expectedReplaceAllTrue.csv
@@ -0,0 +1,2 @@
+density;sensorId
+122.0;aenaor01
diff --git
a/ui/cypress/fixtures/connect/regexRule/expectedReplaceAllTrueWithEmptyString.csv
b/ui/cypress/fixtures/connect/regexRule/expectedReplaceAllTrueWithEmptyString.csv
new file mode 100644
index 0000000000..91547295c5
--- /dev/null
+++
b/ui/cypress/fixtures/connect/regexRule/expectedReplaceAllTrueWithEmptyString.csv
@@ -0,0 +1,2 @@
+density;sensorId
+122.0;enor01
diff --git a/ui/cypress/fixtures/connect/regexRule/input.csv
b/ui/cypress/fixtures/connect/regexRule/input.csv
new file mode 100644
index 0000000000..78eb36e07f
--- /dev/null
+++ b/ui/cypress/fixtures/connect/regexRule/input.csv
@@ -0,0 +1,2 @@
+timestamp;density;sensorId
+1720018277000;122.0;sensor01
diff --git a/ui/cypress/support/utils/connect/ConnectEventSchemaUtils.ts
b/ui/cypress/support/utils/connect/ConnectEventSchemaUtils.ts
index 6bcfe2b231..b8cdfeabeb 100644
--- a/ui/cypress/support/utils/connect/ConnectEventSchemaUtils.ts
+++ b/ui/cypress/support/utils/connect/ConnectEventSchemaUtils.ts
@@ -255,6 +255,10 @@ export class ConnectEventSchemaUtils {
cy.get('#event-schema-next-button').parent().should('not.be.disabled');
}
+ public static schemaPreviewResultEvent() {
+ return cy.dataCy('schema-preview-result-event', { timeout: 10000 });
+ }
+
public static finishEventSchemaConfiguration() {
// Click next
cy.dataCy('sp-connect-schema-editor', { timeout: 10000 }).should(
@@ -270,7 +274,18 @@ export class ConnectEventSchemaUtils {
ConnectEventSchemaUtils.validateRuntimeName(propertyName);
}
- //
+ public static regexValueInput() {
+ return cy.dataCy('regex-value');
+ }
+
+ public static regexReplaceWithValueInput() {
+ return cy.dataCy('regex-replace-with-value');
+ }
+
+ public static regexReplaceAllCheckbox() {
+ return cy.dataCy('regex-replace-all-value');
+ }
+
/**
* Function to escape special characters in a string for use in Cypress
* selectors
diff --git a/ui/cypress/tests/connect/rules/deleteTransformationRule.spec.ts
b/ui/cypress/tests/connect/rules/deleteTransformationRule.spec.ts
index ad558ab8d2..19efb96b46 100644
--- a/ui/cypress/tests/connect/rules/deleteTransformationRule.spec.ts
+++ b/ui/cypress/tests/connect/rules/deleteTransformationRule.spec.ts
@@ -37,7 +37,7 @@ describe('Connect delete rule transformation', () => {
ConnectEventSchemaUtils.deleteProperty('reserved bit_1');
ConnectEventSchemaUtils.deleteProperty('reserved bit_2');
- cy.dataCy('schema-preview-result-event').should(
+ ConnectEventSchemaUtils.schemaPreviewResultEvent().should(
'have.text',
'{\u00A0\u00A0\u00A0\u00A0"timestamp":\u00A01715356080000}',
);
@@ -69,7 +69,7 @@ describe('Connect delete rule transformation', () => {
ConnectEventSchemaUtils.deleteProperty('child');
// The resulting string contains non-breaking spaces character (\u00A0)
- cy.dataCy('schema-preview-result-event').should(
+ ConnectEventSchemaUtils.schemaPreviewResultEvent().should(
'have.text',
'{\u00A0\u00A0\u00A0\u00A0"parent":\u00A0{\u00A0\u00A0\u00A0\u00A0\u00A0\u00A0\u00A0\u00A0"child_two":\u00A0"textTwo"\u00A0\u00A0\u00A0\u00A0},\u00A0\u00A0\u00A0\u00A0"timestamp":\u00A01667904471000}',
);
@@ -78,7 +78,7 @@ describe('Connect delete rule transformation', () => {
// Test to delete the parent property
ConnectEventSchemaUtils.deleteProperty('parent');
- cy.dataCy('schema-preview-result-event').should(
+ ConnectEventSchemaUtils.schemaPreviewResultEvent().should(
'have.text',
'{\u00A0\u00A0\u00A0\u00A0"timestamp":\u00A01667904471000}',
@@ -88,7 +88,7 @@ describe('Connect delete rule transformation', () => {
ConnectEventSchemaUtils.deleteProperty('child');
ConnectEventSchemaUtils.deleteProperty('child_two');
- cy.dataCy('schema-preview-result-event').should(
+ ConnectEventSchemaUtils.schemaPreviewResultEvent().should(
'have.text',
'{\u00A0\u00A0\u00A0\u00A0"parent":\u00A0{},\u00A0\u00A0\u00A0\u00A0"timestamp":\u00A01667904471000}',
);
@@ -97,7 +97,7 @@ describe('Connect delete rule transformation', () => {
// Test to delete the parent property
ConnectEventSchemaUtils.deleteProperty('parent');
- cy.dataCy('schema-preview-result-event').should(
+ ConnectEventSchemaUtils.schemaPreviewResultEvent().should(
'have.text',
'{\u00A0\u00A0\u00A0\u00A0"timestamp":\u00A01667904471000}',
@@ -109,7 +109,7 @@ describe('Connect delete rule transformation', () => {
ConnectEventSchemaUtils.deleteProperty('child');
ConnectEventSchemaUtils.deleteProperty('child_two');
- cy.dataCy('schema-preview-result-event').should(
+ ConnectEventSchemaUtils.schemaPreviewResultEvent().should(
'have.text',
'{\u00A0\u00A0\u00A0\u00A0"parent":\u00A0{},\u00A0\u00A0\u00A0\u00A0"timestamp":\u00A01667904471000}',
);
diff --git a/ui/cypress/tests/connect/rules/regexRule.spec.ts
b/ui/cypress/tests/connect/rules/regexRule.spec.ts
new file mode 100644
index 0000000000..c9ac4d194a
--- /dev/null
+++ b/ui/cypress/tests/connect/rules/regexRule.spec.ts
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { ConnectUtils } from '../../../support/utils/connect/ConnectUtils';
+import { ConnectEventSchemaUtils } from
'../../../support/utils/connect/ConnectEventSchemaUtils';
+import { ConnectBtns } from '../../../support/utils/connect/ConnectBtns';
+import { FileManagementUtils } from
'../../../support/utils/FileManagementUtils';
+
+describe('Connect value rule transformations', () => {
+ beforeEach('Setup Test', () => {
+ cy.initStreamPipesTest();
+ FileManagementUtils.addFile('connect/regexRule/input.csv');
+ });
+
+ it('Perform Test Replace All True', () => {
+ runTest('aenaor01', 'a', true, 'expectedReplaceAllTrue.csv');
+ });
+
+ it('Perform Test Replace All False', () => {
+ runTest('aensor01', 'a', false, 'expectedReplaceAllFalse.csv');
+ });
+
+ it('Perform Test Replace With Empty String', () => {
+ runTest(
+ 'enor01',
+ '',
+ true,
+ 'expectedReplaceAllTrueWithEmptyString.csv',
+ );
+ });
+
+ function runTest(
+ expectedPreview: string,
+ replaceWith: string,
+ replaceAll: boolean,
+ expecedCsvFile: string,
+ ) {
+ const adapterConfiguration =
+ ConnectUtils.setUpPreprocessingRuleTest(false);
+
+ // Ensure regex input is not shown for number property
+ ConnectEventSchemaUtils.clickEditProperty('density');
+ ConnectEventSchemaUtils.regexValueInput().should('not.exist');
+ ConnectBtns.saveEditProperty().click();
+ ConnectEventSchemaUtils.clickEditProperty('sensorId');
+ ConnectEventSchemaUtils.regexValueInput().should('be.visible');
+
+ // Validate that preview works
+ ConnectEventSchemaUtils.regexValueInput().type('s');
+ if (replaceWith !== '') {
+ ConnectEventSchemaUtils.regexReplaceWithValueInput().type(
+ replaceWith,
+ );
+ }
+ if (replaceAll) {
+ ConnectEventSchemaUtils.regexReplaceAllCheckbox().click();
+ }
+
+ ConnectBtns.saveEditProperty().click();
+
+ ConnectEventSchemaUtils.schemaPreviewResultEvent().should(
+ 'contain.text',
+ expectedPreview,
+ );
+ ConnectEventSchemaUtils.markPropertyAsTimestamp('timestamp');
+ ConnectEventSchemaUtils.finishEventSchemaConfiguration();
+
+ ConnectUtils.tearDownPreprocessingRuleTest(
+ adapterConfiguration,
+ 'cypress/fixtures/connect/regexRule/' + expecedCsvFile,
+ true,
+ );
+ }
+});
diff --git
a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model-client.ts
b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model-client.ts
index 4cbe0f01c0..a71be226f2 100644
---
a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model-client.ts
+++
b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model-client.ts
@@ -16,13 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
-
/* tslint:disable */
/* eslint-disable */
// @ts-nocheck
-// Generated using typescript-generator version 3.2.1263 on 2024-09-08
22:18:46.
+// Generated using typescript-generator version 3.2.1263 on 2024-10-07
13:58:29.
-import { Storable } from './streampipes-model';
+import { Storable } from '@streampipes/platform-services';
export class Group implements Storable {
elementId: string;
diff --git
a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts
b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts
index 1ae8e4e32d..e5faa9118d 100644
---
a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts
+++
b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts
@@ -16,11 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-
/* tslint:disable */
/* eslint-disable */
// @ts-nocheck
-// Generated using typescript-generator version 3.2.1263 on 2024-09-30
14:11:40.
+// Generated using typescript-generator version 3.2.1263 on 2024-10-07
13:58:27.
export class NamedStreamPipesEntity implements Storable {
'@class':
@@ -209,6 +208,7 @@ export class TransformationRuleDescription {
|
'org.apache.streampipes.model.connect.rules.value.AddValueTransformationRuleDescription'
|
'org.apache.streampipes.model.connect.rules.value.TimestampTranfsformationRuleDescription'
|
'org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription'
+ |
'org.apache.streampipes.model.connect.rules.value.RegexTransformationRuleDescription'
|
'org.apache.streampipes.model.connect.rules.value.ChangeDatatypeTransformationRuleDescription'
|
'org.apache.streampipes.model.connect.rules.value.CorrectionValueTransformationRuleDescription'
|
'org.apache.streampipes.model.connect.rules.stream.StreamTransformationRuleDescription'
@@ -261,6 +261,8 @@ export class TransformationRuleDescription {
return DeleteRuleDescription.fromData(data);
case
'org.apache.streampipes.model.connect.rules.schema.RenameRuleDescription':
return RenameRuleDescription.fromData(data);
+ case
'org.apache.streampipes.model.connect.rules.value.RegexTransformationRuleDescription':
+ return RegexTransformationRuleDescription.fromData(data);
case
'org.apache.streampipes.model.connect.rules.schema.MoveRuleDescription':
return MoveRuleDescription.fromData(data);
case
'org.apache.streampipes.model.connect.rules.value.ChangeDatatypeTransformationRuleDescription':
@@ -282,6 +284,7 @@ export class ValueTransformationRuleDescription extends
TransformationRuleDescri
|
'org.apache.streampipes.model.connect.rules.value.AddValueTransformationRuleDescription'
|
'org.apache.streampipes.model.connect.rules.value.TimestampTranfsformationRuleDescription'
|
'org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription'
+ |
'org.apache.streampipes.model.connect.rules.value.RegexTransformationRuleDescription'
|
'org.apache.streampipes.model.connect.rules.value.ChangeDatatypeTransformationRuleDescription'
|
'org.apache.streampipes.model.connect.rules.value.CorrectionValueTransformationRuleDescription';
@@ -316,6 +319,8 @@ export class ValueTransformationRuleDescription extends
TransformationRuleDescri
return CorrectionValueTransformationRuleDescription.fromData(
data,
);
+ case
'org.apache.streampipes.model.connect.rules.value.RegexTransformationRuleDescription':
+ return RegexTransformationRuleDescription.fromData(data);
}
}
}
@@ -3061,6 +3066,30 @@ export class QuantitativeValue extends
ValueSpecification {
}
}
+export class RegexTransformationRuleDescription extends
ValueTransformationRuleDescription {
+ '@class':
'org.apache.streampipes.model.connect.rules.value.RegexTransformationRuleDescription';
+ 'regex': string;
+ 'replaceAll': boolean;
+ 'replaceWith': string;
+ 'runtimeKey': string;
+
+ static 'fromData'(
+ data: RegexTransformationRuleDescription,
+ target?: RegexTransformationRuleDescription,
+ ): RegexTransformationRuleDescription {
+ if (!data) {
+ return data;
+ }
+ const instance = target || new RegexTransformationRuleDescription();
+ super.fromData(data, instance);
+ instance.regex = data.regex;
+ instance.replaceAll = data.replaceAll;
+ instance.replaceWith = data.replaceWith;
+ instance.runtimeKey = data.runtimeKey;
+ return instance;
+ }
+}
+
export class RemoveDuplicatesTransformationRuleDescription extends
StreamTransformationRuleDescription {
'@class':
'org.apache.streampipes.model.connect.rules.stream.RemoveDuplicatesTransformationRuleDescription';
'filterTimeWindow': string;
@@ -4080,6 +4109,7 @@ export type TransformationRuleDescriptionUnion =
| CreateNestedRuleDescription
| DeleteRuleDescription
| RenameRuleDescription
+ | RegexTransformationRuleDescription
| MoveRuleDescription
| ChangeDatatypeTransformationRuleDescription
| CorrectionValueTransformationRuleDescription;
@@ -4100,7 +4130,8 @@ export type ValueTransformationRuleDescriptionUnion =
| AddValueTransformationRuleDescription
| TimestampTranfsformationRuleDescription
| UnitTransformRuleDescription
- | CorrectionValueTransformationRuleDescription;
+ | CorrectionValueTransformationRuleDescription
+ | RegexTransformationRuleDescription;
function __getCopyArrayFn<T>(itemCopyFn: (item: T) => T): (array: T[]) => T[] {
return (array: T[]) => __copyArray(array, itemCopyFn);
diff --git
a/ui/projects/streampipes/platform-services/src/lib/model/types/data-type.ts
b/ui/projects/streampipes/platform-services/src/lib/model/types/data-type.ts
index 077b9b9741..74e75ac9ea 100644
--- a/ui/projects/streampipes/platform-services/src/lib/model/types/data-type.ts
+++ b/ui/projects/streampipes/platform-services/src/lib/model/types/data-type.ts
@@ -41,4 +41,8 @@ export class DataType {
public static isBooleanType(datatype: string): boolean {
return datatype === DataType.BOOLEAN;
}
+
+ public static isStringType(datatype: string): boolean {
+ return datatype === DataType.STRING;
+ }
}
diff --git a/ui/src/app/connect/connect.module.ts
b/ui/src/app/connect/connect.module.ts
index 0d0e598a31..f66f7edebb 100644
--- a/ui/src/app/connect/connect.module.ts
+++ b/ui/src/app/connect/connect.module.ts
@@ -104,6 +104,7 @@ import { AdapterStatusLightComponent } from
'./components/existing-adapters/adap
import { SpAdapterDeploymentSettingsComponent } from
'./components/adapter-configuration/adapter-settings/adapter-deployment-settings/adapter-deployment-settings.component';
import { SpAdapterDocumentationDialogComponent } from
'./dialog/adapter-documentation/adapter-documentation-dialog.component';
import { AdapterDetailsDataComponent } from
'./components/adapter-details/adapter-details-data/adapter-details-data.component';
+import { EditRegexTransformationComponent } from
'./dialog/edit-event-property/components/edit-regex-transformation/edit-regex-transformation.component';
@NgModule({
imports: [
@@ -204,6 +205,7 @@ import { AdapterDetailsDataComponent } from
'./components/adapter-details/adapte
DeleteAdapterDialogComponent,
EventSchemaComponent,
EditEventPropertyComponent,
+ EditRegexTransformationComponent,
EventPropertyRowComponent,
EditUnitTransformationComponent,
EditSchemaTransformationComponent,
diff --git
a/ui/src/app/connect/dialog/edit-event-property/components/edit-regex-transformation/edit-regex-transformation.component.html
b/ui/src/app/connect/dialog/edit-event-property/components/edit-regex-transformation/edit-regex-transformation.component.html
new file mode 100644
index 0000000000..e75269a4c2
--- /dev/null
+++
b/ui/src/app/connect/dialog/edit-event-property/components/edit-regex-transformation/edit-regex-transformation.component.html
@@ -0,0 +1,52 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+<div fxLayout="column" fxLayoutAlign="start">
+ <div fxFlex="50" fxLayoutAlign="start center">
+ <mat-form-field class="w-100" color="accent">
+ <input
+ matInput
+ type="text"
+ name="regex"
+ placeholder="Regular expression"
+ data-cy="regex-value"
+ [(ngModel)]="cachedProperty.additionalMetadata.regex"
+ />
+ </mat-form-field>
+ </div>
+ <div fxFlex="50" fxLayoutAlign="start center">
+ <mat-form-field class="w-100" color="accent">
+ <input
+ matInput
+ type="text"
+ name="replaceWith"
+ placeholder="Replace with"
+ data-cy="regex-replace-with-value"
+ [(ngModel)]="cachedProperty.additionalMetadata.replaceWith"
+ />
+ </mat-form-field>
+ </div>
+ <div fxFlex="50" fxLayoutAlign="start center">
+ <mat-checkbox
+ data-cy="regex-replace-all-value"
+ [(ngModel)]="cachedProperty.additionalMetadata.replaceAll"
+ >
+ <span>Replace All Occurrences</span>
+ </mat-checkbox>
+ </div>
+</div>
diff --git
a/ui/src/app/connect/dialog/edit-event-property/components/edit-regex-transformation/edit-regex-transformation.component.scss
b/ui/src/app/connect/dialog/edit-event-property/components/edit-regex-transformation/edit-regex-transformation.component.scss
new file mode 100644
index 0000000000..13cbc4aacb
--- /dev/null
+++
b/ui/src/app/connect/dialog/edit-event-property/components/edit-regex-transformation/edit-regex-transformation.component.scss
@@ -0,0 +1,17 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
diff --git
a/ui/src/app/connect/dialog/edit-event-property/components/edit-regex-transformation/edit-regex-transformation.component.ts
b/ui/src/app/connect/dialog/edit-event-property/components/edit-regex-transformation/edit-regex-transformation.component.ts
new file mode 100644
index 0000000000..f926738e0e
--- /dev/null
+++
b/ui/src/app/connect/dialog/edit-event-property/components/edit-regex-transformation/edit-regex-transformation.component.ts
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { Component, Input } from '@angular/core';
+import { EventPropertyUnion } from '@streampipes/platform-services';
+
+@Component({
+ selector: 'sp-edit-regex-transformation',
+ templateUrl: './edit-regex-transformation.component.html',
+ styleUrl: './edit-regex-transformation.component.scss',
+})
+export class EditRegexTransformationComponent {
+ @Input() cachedProperty: EventPropertyUnion;
+}
diff --git
a/ui/src/app/connect/dialog/edit-event-property/components/edit-value-transformation/edit-value-transformation.component.html
b/ui/src/app/connect/dialog/edit-event-property/components/edit-value-transformation/edit-value-transformation.component.html
index 5986d8137d..9cfb5cc517 100644
---
a/ui/src/app/connect/dialog/edit-event-property/components/edit-value-transformation/edit-value-transformation.component.html
+++
b/ui/src/app/connect/dialog/edit-event-property/components/edit-value-transformation/edit-value-transformation.component.html
@@ -60,4 +60,14 @@
/>
</mat-form-field>
</sp-ep-settings-section>
+ <sp-ep-settings-section
+ sectionTitle="Regex Value Transformation"
+ sectionDescription="Transforms the original field values using the
provided regex"
+ fxLayout="column"
+ fxFlex="100"
+ *ngIf="isStringProperty"
+ >
+ <sp-edit-regex-transformation [cachedProperty]="cachedProperty">
+ </sp-edit-regex-transformation>
+ </sp-ep-settings-section>
</div>
diff --git
a/ui/src/app/connect/dialog/edit-event-property/components/edit-value-transformation/edit-value-transformation.component.ts
b/ui/src/app/connect/dialog/edit-event-property/components/edit-value-transformation/edit-value-transformation.component.ts
index fd87bf99a1..73238f6d3c 100644
---
a/ui/src/app/connect/dialog/edit-event-property/components/edit-value-transformation/edit-value-transformation.component.ts
+++
b/ui/src/app/connect/dialog/edit-event-property/components/edit-value-transformation/edit-value-transformation.component.ts
@@ -40,6 +40,7 @@ export class EditValueTransformationComponent implements
OnInit {
@Input() isListProperty: boolean;
@Input() isPrimitiveProperty: boolean;
@Input() isNumericProperty: boolean;
+ @Input() isStringProperty: boolean;
addedByUser: boolean;
staticValue: string;
diff --git
a/ui/src/app/connect/dialog/edit-event-property/edit-event-property.component.html
b/ui/src/app/connect/dialog/edit-event-property/edit-event-property.component.html
index 0fe15aea50..cc7aad6cad 100644
---
a/ui/src/app/connect/dialog/edit-event-property/edit-event-property.component.html
+++
b/ui/src/app/connect/dialog/edit-event-property/edit-event-property.component.html
@@ -82,6 +82,7 @@
[isNestedProperty]="isEventPropertyNested"
[isListProperty]="isEventPropertyList"
[isPrimitiveProperty]="isEventPropertyPrimitive"
+ [isStringProperty]="isStringProperty"
#valueTransformationComponent
>
</sp-edit-value-transformation>
@@ -98,17 +99,6 @@
</sp-edit-unit-transformation>
</div>
</form>
-
- <!-- <sp-edit-event-property-primitive-->
- <!-- *ngIf="isEventPropertyPrimitive"-->
- <!--
[isTimestampProperty]="isTimestampProperty"-->
- <!-- [cachedProperty]="cachedProperty"-->
- <!--
(isNumericDataType)="isNumericDataType($event)">-->
- <!-- </sp-edit-event-property-primitive>-->
- <!-- <sp-edit-event-property-list-->
- <!-- *ngIf="isEventPropertyList"-->
- <!-- [cachedProperty]="cachedProperty">-->
- <!-- </sp-edit-event-property-list>-->
</div>
</div>
<mat-divider></mat-divider>
diff --git
a/ui/src/app/connect/dialog/edit-event-property/edit-event-property.component.ts
b/ui/src/app/connect/dialog/edit-event-property/edit-event-property.component.ts
index 0cff8aefd4..ea7f191aca 100644
---
a/ui/src/app/connect/dialog/edit-event-property/edit-event-property.component.ts
+++
b/ui/src/app/connect/dialog/edit-event-property/edit-event-property.component.ts
@@ -66,6 +66,7 @@ export class EditEventPropertyComponent implements OnInit {
isEventPropertyNested: boolean;
isEventPropertyList: boolean;
isNumericProperty: boolean;
+ isStringProperty: boolean;
isSaveBtnEnabled: boolean;
private propertyForm: UntypedFormGroup;
@@ -89,6 +90,9 @@ export class EditEventPropertyComponent implements OnInit {
this.isNumericProperty =
SemanticType.isNumber(this.cachedProperty) ||
DataType.isNumberType((this.cachedProperty as any).runtimeType);
+ this.isStringProperty = DataType.isStringType(
+ (this.cachedProperty as any).runtimeType,
+ );
this.createForm();
}
@@ -116,6 +120,13 @@ export class EditEventPropertyComponent implements OnInit {
ep.additionalMetadata.formatString;
result.additionalMetadata.multiplier =
ep.additionalMetadata.multiplier;
+
+ result.additionalMetadata.regex =
+ ep.additionalMetadata.regex || undefined;
+ result.additionalMetadata.replaceWith =
+ ep.additionalMetadata.replaceWith || undefined;
+ result.additionalMetadata.replaceAll =
+ ep.additionalMetadata.replaceAll || undefined;
}
(result as any).staticValue = (ep as any).staticValue;
@@ -177,6 +188,13 @@ export class EditEventPropertyComponent implements OnInit {
this.cachedProperty.additionalMetadata.correctionValue;
this.property.additionalMetadata.operator =
this.cachedProperty.additionalMetadata.operator;
+
+ this.property.additionalMetadata.regex =
+ this.cachedProperty.additionalMetadata.regex;
+ this.property.additionalMetadata.replaceWith =
+ this.cachedProperty.additionalMetadata.replaceWith;
+ this.property.additionalMetadata.replaceAll =
+ this.cachedProperty.additionalMetadata.replaceAll;
}
this.dialogRef.close({ data: this.property });
this.shepherdService.trigger('adapter-field-changed');
diff --git a/ui/src/app/connect/services/transformation-rule.service.ts
b/ui/src/app/connect/services/transformation-rule.service.ts
index b79883961a..5d87c36d69 100644
--- a/ui/src/app/connect/services/transformation-rule.service.ts
+++ b/ui/src/app/connect/services/transformation-rule.service.ts
@@ -31,6 +31,7 @@ import {
EventSchema,
MoveRuleDescription,
PropertyScope,
+ RegexTransformationRuleDescription,
RenameRuleDescription,
SemanticType,
TimestampTranfsformationRuleDescription,
@@ -100,6 +101,13 @@ export class TransformationRuleService {
// Scale
transformationRuleDescriptions = transformationRuleDescriptions
+ .concat(
+ this.getRegexTransformationRules(
+ targetSchema.eventProperties,
+ originalSchema,
+ targetSchema,
+ ),
+ )
.concat(
this.getCorrectionValueRules(
targetSchema.eventProperties,
@@ -671,4 +679,59 @@ export class TransformationRuleService {
return filteredResult;
}
+
+ private getRegexTransformationRules(
+ eventProperties: EventPropertyUnion[],
+ oldEventSchema: EventSchema,
+ newEventSchema: EventSchema,
+ ) {
+ const result: RegexTransformationRuleDescription[] = [];
+
+ eventProperties.forEach(eventProperty => {
+ if (eventProperty instanceof EventPropertyPrimitive) {
+ const newRuntimeName = this.getCompleteRuntimeNameKey(
+ newEventSchema.eventProperties,
+ eventProperty.elementId,
+ );
+
+ if (
+ eventProperty.additionalMetadata?.regex &&
+ eventProperty.additionalMetadata?.regex != ''
+ ) {
+ const rule =
+
this.createRegexTransformationRuleDescriptionFromEventProperty(
+ newRuntimeName,
+ eventProperty,
+ );
+
+ result.push(rule);
+ }
+ } else if (eventProperty instanceof EventPropertyNested) {
+ result.push(
+ ...this.getRegexTransformationRules(
+ eventProperty.eventProperties,
+ oldEventSchema,
+ newEventSchema,
+ ),
+ );
+ }
+ });
+
+ return result;
+ }
+
+ private createRegexTransformationRuleDescriptionFromEventProperty(
+ newRuntimeName: string,
+ eventProperty: EventPropertyPrimitive,
+ ) {
+ const rule: RegexTransformationRuleDescription =
+ new RegexTransformationRuleDescription();
+ rule['@class'] =
+
'org.apache.streampipes.model.connect.rules.value.RegexTransformationRuleDescription';
+ rule.runtimeKey = newRuntimeName;
+ rule.regex = eventProperty.additionalMetadata.regex;
+ rule.replaceWith = eventProperty.additionalMetadata.replaceWith ?? '';
+ rule.replaceAll = eventProperty.additionalMetadata.replaceAll ?? false;
+ return rule;
+ }
}
