This is an automated email from the ASF dual-hosted git repository. zehnder pushed a commit to branch STREAMPIPES-483 in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit 8bb2935235908c50e98e7b77e1c0248f819c242c Author: Philipp Zehnder <[email protected]> AuthorDate: Mon Dec 27 11:19:09 2021 +0100 [STREAMPIPES-483] Add test to validate connect preprocessing rules --- .../fixtures/connect/schemaRules/expected.csv | 2 + ui/cypress/fixtures/connect/schemaRules/input.csv | 2 + ui/cypress/support/utils/AdapterUtils.ts | 16 +++- ui/cypress/support/utils/DataLakeUtils.ts | 17 +++-- ui/cypress/support/utils/FileManagementUtils.ts | 20 ++--- ui/cypress/tests/adapter/schemaRules.smoke.ts | 88 +++++++++++++++------- .../event-property-row.component.html | 9 ++- .../schema-editor-header.component.html | 11 ++- .../edit-data-type/edit-data-type.component.html | 8 +- 9 files changed, 117 insertions(+), 56 deletions(-) diff --git a/ui/cypress/fixtures/connect/schemaRules/expected.csv b/ui/cypress/fixtures/connect/schemaRules/expected.csv new file mode 100644 index 0000000..6f81327 --- /dev/null +++ b/ui/cypress/fixtures/connect/schemaRules/expected.csv @@ -0,0 +1,2 @@ +count;temperature +122.0;11.0 diff --git a/ui/cypress/fixtures/connect/schemaRules/input.csv b/ui/cypress/fixtures/connect/schemaRules/input.csv new file mode 100644 index 0000000..f92f0b8 --- /dev/null +++ b/ui/cypress/fixtures/connect/schemaRules/input.csv @@ -0,0 +1,2 @@ +count;density;temperature +122.0;62.0;11 diff --git a/ui/cypress/support/utils/AdapterUtils.ts b/ui/cypress/support/utils/AdapterUtils.ts index 7eaadff..9697ae3 100644 --- a/ui/cypress/support/utils/AdapterUtils.ts +++ b/ui/cypress/support/utils/AdapterUtils.ts @@ -123,7 +123,7 @@ export class AdapterUtils { cy.get('button').contains('Next').parent().click(); } - private static configureFormat(adapterConfiguration: GenericAdapterInput) { + public static configureFormat(adapterConfiguration: GenericAdapterInput) { // Select format cy.dataCy(adapterConfiguration.format).click(); @@ -142,9 +142,9 @@ export class AdapterUtils { } - private static markPropertyAsTimestamp(propertyName: string) { + public static markPropertyAsTimestamp(propertyName: string) { // Mark property as timestamp - cy.get('#event-schema-next-button').should('be.disabled'); + this.eventSchemaNextBtnDisabled(); // Edit timestamp cy.dataCy('edit-' + propertyName, { timeout: 10000 }).click(); @@ -154,6 +154,14 @@ export class AdapterUtils { // Close cy.dataCy('sp-save-edit-property').click(); + this.eventSchemaNextBtnEnabled(); + } + + public static eventSchemaNextBtnDisabled() { + cy.get('#event-schema-next-button').should('be.disabled'); + } + + public static eventSchemaNextBtnEnabled() { cy.get('#event-schema-next-button').parent().should('not.be.disabled'); } @@ -167,7 +175,7 @@ export class AdapterUtils { AdapterUtils.startAdapter(adapterInput, 'sp-connect-adapter-live-preview'); } - private static startSetAdapter(adapterInput: AdapterInput) { + public static startSetAdapter(adapterInput: AdapterInput) { AdapterUtils.startAdapter(adapterInput, 'sp-connect-adapter-set-success'); } diff --git a/ui/cypress/support/utils/DataLakeUtils.ts b/ui/cypress/support/utils/DataLakeUtils.ts index 35f7421..2dfd50a 100644 --- a/ui/cypress/support/utils/DataLakeUtils.ts +++ b/ui/cypress/support/utils/DataLakeUtils.ts @@ -162,7 +162,7 @@ export class DataLakeUtils { cy.get('div').contains('DataLake').parent().click(); } - public static checkResults(dataLakeIndex: string, fileRoute: string) { + public static checkResults(dataLakeIndex: string, fileRoute: string, ignoreTime: boolean = false) { // Validate result in datalake cy.request({ @@ -177,7 +177,7 @@ export class DataLakeUtils { }).should((response) => { const actualResultString = response.body; cy.readFile(fileRoute).then((expectedResultString) => { - DataLakeUtils.resultEqual(actualResultString, expectedResultString); + DataLakeUtils.resultEqual(actualResultString, expectedResultString, ignoreTime); }); }); } @@ -199,10 +199,17 @@ export class DataLakeUtils { .type(`${value.toLocaleString()}{enter}`); } - private static resultEqual(expected: string, actual: string) { + private static resultEqual(actual: string, expected: string, ignoreTime: boolean) { + let actualResult; + if (ignoreTime) { + actualResult = DataLakeUtils.parseCsv(actual).map(row => { + return row.splice(1); + }); + } else { + actualResult = DataLakeUtils.parseCsv(actual); + } const expectedResult = DataLakeUtils.parseCsv(expected); - const actualResult = DataLakeUtils.parseCsv(actual); - expect(expectedResult).to.deep.equal(actualResult); + expect(actualResult).to.deep.equal(expectedResult); } private static parseCsv(csv: string) { diff --git a/ui/cypress/support/utils/FileManagementUtils.ts b/ui/cypress/support/utils/FileManagementUtils.ts index 13b0d01..581a02c 100644 --- a/ui/cypress/support/utils/FileManagementUtils.ts +++ b/ui/cypress/support/utils/FileManagementUtils.ts @@ -21,24 +21,24 @@ export class FileManagementUtils { public static addFile(filePath: string) { // Go to StreamPipes file management - cy.visit('#/files'); + cy.visit('#/files'); // Open file upload dialog - cy.dataCy('sp-open-file-upload-dialog').click(); + cy.dataCy('sp-open-file-upload-dialog').click(); // Upload file - // const filepath = 'fileTest/test.csv' - cy.dataCy('sp-file-management-file-input').attachFile(filePath); - cy.dataCy('sp-file-management-store-file').click(); + // const filepath = 'fileTest/test.csv' + cy.dataCy('sp-file-management-file-input').attachFile(filePath); + cy.dataCy('sp-file-management-store-file').click(); } public static deleteFile() { // Go to StreamPipes file management - cy.visit('#/files'); + cy.visit('#/files'); // Check if file was uploaded and delete it - cy.dataCy('delete').should('have.length', 1); - cy.dataCy('delete').click(); - cy.dataCy('confirm-delete').click(); - cy.dataCy('delete').should('have.length', 0); + cy.dataCy('delete').should('have.length', 1); + cy.dataCy('delete').click(); + cy.dataCy('confirm-delete').click(); + cy.dataCy('delete').should('have.length', 0); } } diff --git a/ui/cypress/tests/adapter/schemaRules.smoke.ts b/ui/cypress/tests/adapter/schemaRules.smoke.ts index a922e73..d0be54e 100644 --- a/ui/cypress/tests/adapter/schemaRules.smoke.ts +++ b/ui/cypress/tests/adapter/schemaRules.smoke.ts @@ -18,59 +18,89 @@ import { AdapterUtils } from '../../support/utils/AdapterUtils'; import { SpecificAdapterBuilder } from '../../support/builder/SpecificAdapterBuilder'; +import { FileManagementUtils } from '../../support/utils/FileManagementUtils'; +import { GenericAdapterBuilder } from '../../support/builder/GenericAdapterBuilder'; +import { DataLakeUtils } from '../../support/utils/DataLakeUtils'; describe('Test Random Data Simulator Stream Adapter', () => { beforeEach('Setup Test', () => { cy.initStreamPipesTest(); + FileManagementUtils.addFile('connect/schemaRules/input.csv'); }); it('Perform Test', () => { - const adapterConfiguration = SpecificAdapterBuilder - .create('Machine_Data_Simulator') - .setName('Machine Data Simulator Test') - .addInput('input', 'wait-time-ms', '1000') + const adapterConfiguration = GenericAdapterBuilder + .create('File_Set') + .setStoreInDataLake() + .setTimestampProperty('timestamp') + .setName('Adapter to test schema rules') + .setFormat('csv') + .addFormatInput('input', 'delimiter', ';') + .addFormatInput('checkbox', 'header', 'check') .build(); AdapterUtils.goToConnect(); - AdapterUtils.selectAdapter(adapterConfiguration.adapterType); - - AdapterUtils.configureAdapter(adapterConfiguration.adapterConfiguration); + AdapterUtils.configureAdapter(adapterConfiguration.protocolConfiguration); + AdapterUtils.configureFormat(adapterConfiguration); // wait till schema is shown - cy.dataCy('sp-connect-schema-editor', { timeout: 60000 }).should('be.visible'); - // Add fixed property + // Add static value to event + // TODO FIX breaks adapter // Click add a static value to event - cy.dataCy('connect-add-static-property', { timeout: 10000 }).click(); - - // Edit new property - const propertyName = 'staticPropertyName'; - const propertyValue = 'id1'; - cy.dataCy('edit-key_0', { timeout: 10000 }).click(); - - cy.dataCy('connect-edit-field-runtime-name', { timeout: 10000 }) - .type('{backspace}{backspace}{backspace}{backspace}{backspace}' + propertyName); - cy.dataCy('connect-edit-field-static-value', { timeout: 10000 }).type(propertyValue); - - cy.dataCy('sp-save-edit-property').click(); + // cy.dataCy('connect-add-static-property', { timeout: 10000 }).click(); + // + // // Edit new property + // const propertyName = 'staticPropertyName'; + // const propertyValue = 'id1'; + // cy.dataCy('edit-key_0', { timeout: 10000 }).click(); + // + // cy.dataCy('connect-edit-field-runtime-name', { timeout: 10000 }) + // .type('{backspace}{backspace}{backspace}{backspace}{backspace}' + propertyName); + // cy.dataCy('connect-edit-field-static-value', { timeout: 10000 }).type(propertyValue); + // + // cy.dataCy('sp-save-edit-property').click(); + // + // // validate that static value is persisted + // cy.dataCy('edit-' + propertyName.toLowerCase(), { timeout: 10000 }).click(); + // cy.dataCy('connect-edit-field-static-value', { timeout: 10000 }).should('have.value', propertyValue); + // cy.dataCy('sp-save-edit-property').click(); + + // Delete property + cy.dataCy('delete-property-density', { timeout: 10000 }).children().click({ force: true }); + cy.dataCy('connect-schema-delete-properties-btn', { timeout: 10000 }).click(); + + // TODO FIX breaks adapter + // Change data type + // cy.dataCy('edit-temperature', { timeout: 10000 }).click(); + // cy.dataCy('connect-change-runtime-type').click().get('mat-option').contains('Float').click(); + // cy.dataCy('sp-save-edit-property').click(); + // // validate that static value is persisted + // cy.dataCy('edit-temperature', { timeout: 10000 }).click({ force: true }); + // cy.dataCy('connect-change-runtime-type', { timeout: 10000 }).contains('Float'); + // cy.dataCy('sp-save-edit-property').click(); - // validate that static value is persisted - cy.dataCy('edit-' + propertyName, { timeout: 10000 }).click(); - cy.dataCy('connect-edit-field-static-value', { timeout: 10000 }).should('have.value', propertyValue); - cy.dataCy('sp-save-edit-property').click(); + // Add timestamp + AdapterUtils.eventSchemaNextBtnDisabled(); + cy.dataCy('connect-schema-add-timestamp-btn', { timeout: 10000 }).click(); + AdapterUtils.eventSchemaNextBtnEnabled(); AdapterUtils.finishEventSchemaConfiguration(); - AdapterUtils.startAdapter(adapterConfiguration, 'sp-connect-adapter-live-preview'); + AdapterUtils.startSetAdapter(adapterConfiguration); - // Add timestamp - // Delete - // Change data type + // Wait till data is stored + cy.wait(10000); + + DataLakeUtils.checkResults( + 'adaptertotestschemarules', + 'cypress/fixtures/connect/schemaRules/expected.csv', + true); }); }); diff --git a/ui/src/app/connect/components/schema-editor/event-property-row/event-property-row.component.html b/ui/src/app/connect/components/schema-editor/event-property-row/event-property-row.component.html index 8fda374..df17659 100644 --- a/ui/src/app/connect/components/schema-editor/event-property-row/event-property-row.component.html +++ b/ui/src/app/connect/components/schema-editor/event-property-row/event-property-row.component.html @@ -47,15 +47,18 @@ *ngIf="isNested || isPrimitive || isList"> <button [disabled]="!isEditable" color="accent" mat-button (click)="openEditDialog(node.data)" - [attr.data-cy]="'edit-' + label"> + [attr.data-cy]="'edit-' + label.toLowerCase()"> <mat-icon>edit</mat-icon> </button> </div> <div fxFlex="0 1 auto" fxLayoutAlign="center center"> <mat-checkbox *ngIf="isNested || isPrimitive || isList" - (click)="selectProperty(node.data.id, undefined)" [disabled]="!isEditable" - [class.checkbox-selected]="node.data.selected" [checked]="node.data.selected"> + (click)="selectProperty(node.data.id, undefined)" + [disabled]="!isEditable" + [class.checkbox-selected]="node.data.selected" + [checked]="node.data.selected" + [attr.data-cy]="'delete-property-' + label.toLowerCase()"> </mat-checkbox> </div> </div> diff --git a/ui/src/app/connect/components/schema-editor/schema-editor-header/schema-editor-header.component.html b/ui/src/app/connect/components/schema-editor/schema-editor-header/schema-editor-header.component.html index b3cae9a..2c30634 100644 --- a/ui/src/app/connect/components/schema-editor/schema-editor-header/schema-editor-header.component.html +++ b/ui/src/app/connect/components/schema-editor/schema-editor-header/schema-editor-header.component.html @@ -34,7 +34,8 @@ <button color="primary" mat-button data-cy="connect-add-timestamp-property" - (click)=addTimestampProperty()> + (click)=addTimestampProperty() + data-cy="connect-schema-add-timestamp-btn"> <mat-icon matTooltip="Add timestamp to event schema">access_time</mat-icon> </button> <button color="primary" @@ -53,8 +54,12 @@ </sp-error-hint> - <button style="padding-right:0" color="primary" mat-button [disabled]="countSelected == 0" - (click)="removeSelectedProperties()"> + <button style="padding-right:0" + color="primary" + mat-button + [disabled]="countSelected == 0" + (click)="removeSelectedProperties()" + data-cy="connect-schema-delete-properties-btn"> <mat-icon matTooltip="Remove selected Properties">delete</mat-icon> </button> </div> diff --git a/ui/src/app/connect/dialog/edit-event-property/components/edit-data-type/edit-data-type.component.html b/ui/src/app/connect/dialog/edit-event-property/components/edit-data-type/edit-data-type.component.html index f8da0c0..7e8c9f4 100644 --- a/ui/src/app/connect/dialog/edit-event-property/components/edit-data-type/edit-data-type.component.html +++ b/ui/src/app/connect/dialog/edit-event-property/components/edit-data-type/edit-data-type.component.html @@ -17,8 +17,12 @@ --> <mat-form-field class="doubleWidth" color="accent"> - <mat-select placeholder="Runtime Type" [(ngModel)]="cachedProperty.runtimeType" (selectionChange)="valueChanged()"> - <mat-option *ngFor="let dataType of runtimeDataTypes" [value]="dataType.url"> + <mat-select placeholder="Runtime Type" + [(ngModel)]="cachedProperty.runtimeType" + (selectionChange)="valueChanged()" + data-cy="connect-change-runtime-type"> + <mat-option *ngFor="let dataType of runtimeDataTypes" + [value]="dataType.url"> {{ dataType.label }} </mat-option> </mat-select>
