METRON-1231 Separate Sensor name and topic in the Management UI (merrimanr) closes apache/metron#786
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/6afd95fa Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/6afd95fa Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/6afd95fa Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: 6afd95fabf870424177384538ce4685e77fa585c Parents: 2dd01b1 Author: merrimanr <merrim...@gmail.com> Authored: Tue Jan 16 08:35:42 2018 -0600 Committer: merrimanr <merrim...@apache.org> Committed: Tue Jan 16 08:35:42 2018 -0600 ---------------------------------------------------------------------- .../app/model/sensor-parser-config-history.ts | 1 + .../sensor-parser-config-readonly.component.ts | 2 +- .../sensor-parser-config.component.html | 23 ++++-- .../sensor-parser-config.component.spec.ts | 57 ++++++++++++--- .../sensor-parser-config.component.ts | 44 +++++++---- .../sensor-parser-list.component.html | 12 +-- .../sensor-parser-list.component.spec.ts | 74 ++++++++++--------- .../sensor-parser-list.component.ts | 77 ++++++++++---------- .../sensor-parser-config.service.spec.ts | 4 +- .../app/service/sensor-parser-config.service.ts | 34 ++++----- metron-interface/metron-rest/README.md | 3 +- .../SensorParserConfigController.java | 12 +-- .../rest/service/SensorParserConfigService.java | 4 +- .../impl/SensorParserConfigServiceImpl.java | 10 +-- ...orParserConfigControllerIntegrationTest.java | 67 +++++++++-------- .../StormControllerIntegrationTest.java | 2 +- .../impl/SensorParserConfigServiceImplTest.java | 10 +-- 17 files changed, 252 insertions(+), 184 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/6afd95fa/metron-interface/metron-config/src/app/model/sensor-parser-config-history.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-config/src/app/model/sensor-parser-config-history.ts b/metron-interface/metron-config/src/app/model/sensor-parser-config-history.ts index 0aed6fe..4854001 100644 --- a/metron-interface/metron-config/src/app/model/sensor-parser-config-history.ts +++ b/metron-interface/metron-config/src/app/model/sensor-parser-config-history.ts @@ -17,6 +17,7 @@ */ import {SensorParserConfig} from './sensor-parser-config'; export class SensorParserConfigHistory { + sensorName: string; createdBy: string; modifiedBy: string; createdDate: string; http://git-wip-us.apache.org/repos/asf/metron/blob/6afd95fa/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts b/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts index f28a206..5db6d45 100644 --- a/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts +++ b/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts @@ -330,7 +330,7 @@ export class SensorParserConfigReadonlyComponent implements OnInit { this.sensorParserConfigService.deleteSensorParserConfig(name).subscribe(result => { this.metronAlerts.showSuccessMessage('Deleted sensor ' + name); this.toggleStartStopInProgress(); - this.sensorParserConfigService.dataChangedSource.next([this.sensorParserConfigHistory.config]); + this.sensorParserConfigService.dataChangedSource.next([name]); this.goBack(); }, error => { http://git-wip-us.apache.org/repos/asf/metron/blob/6afd95fa/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.html ---------------------------------------------------------------------- diff --git a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.html b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.html index 186e221..a784436 100644 --- a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.html +++ b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.html @@ -41,16 +41,25 @@ <div class="metron-slider-pane-edit fill load-right-to-left dialog1x" style="overflow: auto" > <div style="height:100%"> - <div class="form-title">{{sensorParserConfig.sensorTopic}} </div> + <div class="form-title">{{sensorName}} </div> <i class="fa fa-times pull-right main close-button" aria-hidden="true" (click)="goBack()"></i> <form role="form" [formGroup]="sensorConfigForm"> <div class="form-group"> - <label attr.for="sensorTopic">NAME * </label> - <input type="text" class="form-control" name="sensorTopic" formControlName="sensorTopic" [(ngModel)]="sensorParserConfig.sensorTopic" (ngModelChange)="onSetSensorName()" [readonly]="editMode"> - <label *ngIf="currentKafkaStatus !== null && currentKafkaStatus === kafkaStatus.NO_TOPIC"><span class="warning-text"> No Matching Kafka Topic </span></label> - <label *ngIf="currentKafkaStatus !== null && currentKafkaStatus === kafkaStatus.EMITTING" ><span class="success-text-color"> Kafka Topic Exists. Emitting </span></label> - <label *ngIf="currentKafkaStatus !== null && currentKafkaStatus === kafkaStatus.NOT_EMITTING"><span class="success-text-color"> Kafka Topic Exists. </span><span class="warning-text" > Not Emitting </span></label> + <label attr.for="sensorName">NAME * </label> + <input type="text" class="form-control" name="sensorName" formControlName="sensorName" [(ngModel)]="sensorName" (ngModelChange)="onSetSensorName()" [readonly]="editMode"> + <div *ngIf="!sensorNameUnique"><label ><span class="warning-text"> Sensor already exists </span></label></div> + </div> + + <div class="form-group"> + <label attr.for="sensorTopic">KAFKA TOPIC </label> + <input type="text" class="form-control" name="sensorTopic" formControlName="sensorTopic" [(ngModel)]="sensorParserConfig.sensorTopic" (ngModelChange)="onSetKafkaTopic()"> + <div *ngIf="!kafkaTopicValid"><label ><span class="warning-text"> Kafka Topic Name Is Invalid </span></label></div> + <div *ngIf="kafkaTopicValid"> + <label *ngIf="currentKafkaStatus !== null && currentKafkaStatus === kafkaStatus.NO_TOPIC"><span class="warning-text"> No Matching Kafka Topic </span></label> + <label *ngIf="currentKafkaStatus !== null && currentKafkaStatus === kafkaStatus.EMITTING" ><span class="success-text-color"> Kafka Topic Exists. Emitting </span></label> + <label *ngIf="currentKafkaStatus !== null && currentKafkaStatus === kafkaStatus.NOT_EMITTING"><span class="success-text-color"> Kafka Topic Exists. </span><span class="warning-text" > Not Emitting </span></label> + </div> </div> <div class="form-group"> @@ -187,7 +196,7 @@ <div class="form-group"> <div class="form-seperator-edit"></div> <div class="button-row"> - <button type="submit" class="btn save-button" [ngClass]="{'disabled':!configValid}" (click)="onSave()">SAVE</button> + <button type="submit" class="btn save-button" [ngClass]="{'disabled':!configValid}" [disabled]="!configValid" (click)="onSave()">SAVE</button> <button class="btn form-enable-disable-button" (click)="goBack()" >CANCEL</button> <span class="advanced-link" [hidden]="showAdvancedParserConfiguration" (click)="showAdvancedParserConfiguration = true">Advanced</span> </div> http://git-wip-us.apache.org/repos/asf/metron/blob/6afd95fa/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.spec.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.spec.ts b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.spec.ts index 6c4eab1..d5f3d67 100644 --- a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.spec.ts +++ b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.spec.ts @@ -64,6 +64,7 @@ class MockActivatedRoute { } class MockSensorParserConfigService extends SensorParserConfigService { + private name: string; private sensorParserConfig: SensorParserConfig; private parsedMessage: any; private postedSensorParserConfig: SensorParserConfig; @@ -73,7 +74,7 @@ class MockSensorParserConfigService extends SensorParserConfigService { super(http2, config2); } - public post(sensorParserConfig: SensorParserConfig): Observable<SensorParserConfig> { + public post(name: string, sensorParserConfig: SensorParserConfig): Observable<SensorParserConfig> { if (this.throwError) { let error = new RestError(); error.message = 'SensorParserConfig post error'; @@ -93,6 +94,15 @@ class MockSensorParserConfigService extends SensorParserConfigService { }); } + public getAll(): Observable<{}> { + return Observable.create(observer => { + let results = {}; + results[this.name] = this.sensorParserConfig; + observer.next(results); + observer.complete(); + }); + } + public getAvailableParsers(): Observable<{}> { return Observable.create(observer => { observer.next({ @@ -110,8 +120,9 @@ class MockSensorParserConfigService extends SensorParserConfigService { }); } - public setSensorParserConfig(result: any) { - this.sensorParserConfig = result; + public setSensorParserConfig(name: string, sensorParserConfig: SensorParserConfig) { + this.name = name; + this.sensorParserConfig = sensorParserConfig; } public setParsedMessage(parsedMessage: any) { @@ -570,7 +581,7 @@ describe('Component: SensorParserConfig', () => { component.sensorParserConfig = Object.assign(new SensorParserConfig(), squidSensorParserConfig); component.createForms(); - expect(Object.keys(component.sensorConfigForm.controls).length).toEqual(15); + expect(Object.keys(component.sensorConfigForm.controls).length).toEqual(16); expect(Object.keys(component.transformsValidationForm.controls).length).toEqual(2); expect(component.showAdvancedParserConfiguration).toEqual(true); @@ -594,6 +605,7 @@ describe('Component: SensorParserConfig', () => { })); it('should init', async(() => { + sensorParserConfigService.setSensorParserConfig('squid', squidSensorParserConfig); component.init('new'); let expectedSensorParserConfig = new SensorParserConfig(); @@ -602,10 +614,11 @@ describe('Component: SensorParserConfig', () => { expect(component.sensorEnrichmentConfig).toEqual(new SensorEnrichmentConfig()); expect(component.indexingConfigurations).toEqual(new IndexingConfigurations()); expect(component.editMode).toEqual(false); + expect(component.currentSensors).toEqual(['squid']); spyOn(component, 'getKafkaStatus'); let sensorParserConfig = Object.assign(new SensorParserConfig(), squidSensorParserConfig); - sensorParserConfigService.setSensorParserConfig(sensorParserConfig); + sensorParserConfigService.setSensorParserConfig('squid', sensorParserConfig); sensorEnrichmentConfigService.setSensorEnrichmentConfig('squid', Object.assign(new SensorEnrichmentConfig(), squidSensorEnrichmentConfig)); sensorIndexingConfigService.setSensorIndexingConfig('squid', @@ -646,7 +659,7 @@ describe('Component: SensorParserConfig', () => { sensorParserConfig = new SensorParserConfig(); sensorParserConfig.sensorTopic = 'bro'; - sensorParserConfigService.setSensorParserConfig(sensorParserConfig); + sensorParserConfigService.setSensorParserConfig('bro', sensorParserConfig); component.showAdvancedParserConfiguration = false; component.init('bro'); @@ -664,18 +677,39 @@ describe('Component: SensorParserConfig', () => { fixture.destroy(); })); - it('should handle onSetSensorName', async(() => { + it('should handle onSetKafkaTopic', async(() => { spyOn(component, 'getKafkaStatus'); spyOn(component, 'isConfigValid'); - component.onSetSensorName(); + component.onSetKafkaTopic(); expect(component.getKafkaStatus).not.toHaveBeenCalled(); expect(component.isConfigValid).toHaveBeenCalled(); component.sensorParserConfig.sensorTopic = 'bro'; + component.onSetKafkaTopic(); + expect(component.kafkaTopicValid).toEqual(true); + expect(component.getKafkaStatus).toHaveBeenCalled(); + + fixture.destroy(); + })); + + it('should handle onSetSensorName', async(() => { + spyOn(component, 'isConfigValid'); + + component.onSetSensorName(); + expect(component.isConfigValid).toHaveBeenCalled(); + expect(component.sensorNameValid).toEqual(false); + + component.sensorName = 'squid'; + component.currentSensors = ['squid']; + component.onSetSensorName(); + expect(component.sensorNameUnique).toEqual(false); + expect(component.sensorNameValid).toEqual(false); + + component.sensorName = 'squidUnique'; component.onSetSensorName(); + expect(component.sensorNameUnique).toEqual(true); expect(component.sensorNameValid).toEqual(true); - expect(component.getKafkaStatus).toHaveBeenCalled(); fixture.destroy(); })); @@ -719,6 +753,7 @@ describe('Component: SensorParserConfig', () => { expect(component.configValid).toEqual(false); component.sensorNameValid = true; + component.kafkaTopicValid = true; component.parserClassValid = true; component.isConfigValid(); @@ -770,7 +805,7 @@ describe('Component: SensorParserConfig', () => { })); it('should handle onSaveGrokStatement', async(() => { - component.sensorParserConfig.sensorTopic = 'squid'; + component.sensorName = 'squid'; component.onSaveGrokStatement('grok statement'); expect(component.grokStatement).toEqual('grok statement'); @@ -979,7 +1014,7 @@ describe('Component: SensorParserConfig', () => { it('should handle onShowGrokPane', async(() => { spyOn(component, 'showPane'); - component.sensorParserConfig.sensorTopic = 'squid'; + component.sensorName = 'squid'; component.onShowGrokPane(); expect(component.patternLabel).toEqual('SQUID'); http://git-wip-us.apache.org/repos/asf/metron/blob/6afd95fa/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.ts b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.ts index 679d057..647e02f 100644 --- a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.ts +++ b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.ts @@ -52,6 +52,7 @@ export class SensorParserConfigComponent implements OnInit { sensorConfigForm: FormGroup; transformsValidationForm: FormGroup; + sensorName: string = ''; sensorParserConfig: SensorParserConfig = new SensorParserConfig(); sensorEnrichmentConfig: SensorEnrichmentConfig = new SensorEnrichmentConfig(); indexingConfigurations: IndexingConfigurations = new IndexingConfigurations(); @@ -66,12 +67,15 @@ export class SensorParserConfigComponent implements OnInit { configValid = false; sensorNameValid = false; + sensorNameUnique = true; + kafkaTopicValid = false; parserClassValid = false; grokStatementValid = false; availableParsers = {}; availableParserNames = []; grokStatement = ''; patternLabel = ''; + currentSensors = []; editMode: boolean = false; @@ -100,6 +104,7 @@ export class SensorParserConfigComponent implements OnInit { init(id: string): void { if (id !== 'new') { this.editMode = true; + this.sensorName = id; this.sensorParserConfigService.get(id).subscribe((results: SensorParserConfig) => { this.sensorParserConfig = results; this.sensorNameValid = true; @@ -144,6 +149,9 @@ export class SensorParserConfigComponent implements OnInit { } else { this.sensorParserConfig = new SensorParserConfig(); this.sensorParserConfig.parserClassName = 'org.apache.metron.parsers.GrokParser'; + this.sensorParserConfigService.getAll().subscribe((results: {}) => { + this.currentSensors = Object.keys(results); + }); } } @@ -159,6 +167,7 @@ export class SensorParserConfigComponent implements OnInit { createSensorConfigForm(): FormGroup { let group: any = {}; + group['sensorName'] = new FormControl(this.sensorName, Validators.required); group['sensorTopic'] = new FormControl(this.sensorParserConfig.sensorTopic, Validators.required); group['parserClassName'] = new FormControl(this.sensorParserConfig.parserClassName, Validators.required); group['grokStatement'] = new FormControl(this.grokStatement); @@ -209,9 +218,16 @@ export class SensorParserConfigComponent implements OnInit { } onSetSensorName(): void { - this.sensorNameValid = this.sensorParserConfig.sensorTopic !== undefined && - this.sensorParserConfig.sensorTopic.length > 0; - if (this.sensorNameValid) { + this.sensorNameUnique = this.currentSensors.indexOf(this.sensorName) === -1; + this.sensorNameValid = this.sensorName !== undefined && + this.sensorName.length > 0 && this.sensorNameUnique; + this.isConfigValid(); + } + + onSetKafkaTopic(): void { + this.kafkaTopicValid = this.sensorParserConfig.sensorTopic !== undefined && + /[a-zA-Z0-9._-]+$/.test(this.sensorParserConfig.sensorTopic); + if (this.kafkaTopicValid) { this.getKafkaStatus(); } this.isConfigValid(); @@ -237,7 +253,7 @@ export class SensorParserConfigComponent implements OnInit { isConfigValid() { let isGrokParser = this.isGrokParser(this.sensorParserConfig); - this.configValid = this.sensorNameValid && this.parserClassValid && (!isGrokParser || this.grokStatementValid); + this.configValid = this.sensorNameValid && this.kafkaTopicValid && this.parserClassValid && (!isGrokParser || this.grokStatementValid); } getKafkaStatus() { @@ -282,7 +298,7 @@ export class SensorParserConfigComponent implements OnInit { this.grokStatement = grokStatement; let grokPath = this.sensorParserConfig.parserConfig['grokPath']; if (!grokPath || grokPath.indexOf('/patterns') === 0) { - this.sensorParserConfig.parserConfig['grokPath'] = '/apps/metron/patterns/' + this.sensorParserConfig.sensorTopic; + this.sensorParserConfig.parserConfig['grokPath'] = '/apps/metron/patterns/' + this.sensorName; } } @@ -293,34 +309,34 @@ export class SensorParserConfigComponent implements OnInit { onSave() { if (!this.indexingConfigurations.hdfs.index) { - this.indexingConfigurations.hdfs.index = this.sensorParserConfig.sensorTopic; + this.indexingConfigurations.hdfs.index = this.sensorName; } if (!this.indexingConfigurations.elasticsearch.index) { - this.indexingConfigurations.elasticsearch.index = this.sensorParserConfig.sensorTopic; + this.indexingConfigurations.elasticsearch.index = this.sensorName; } if (!this.indexingConfigurations.solr.index) { - this.indexingConfigurations.solr.index = this.sensorParserConfig.sensorTopic; + this.indexingConfigurations.solr.index = this.sensorName; } - this.sensorParserConfigService.post(this.sensorParserConfig).subscribe( + this.sensorParserConfigService.post(this.sensorName, this.sensorParserConfig).subscribe( sensorParserConfig => { if (this.isGrokParser(sensorParserConfig)) { this.hdfsService.post(this.sensorParserConfig.parserConfig['grokPath'], this.grokStatement).subscribe( response => {}, (error: RestError) => this.metronAlerts.showErrorMessage(error.message)); } - this.sensorEnrichmentConfigService.post(sensorParserConfig.sensorTopic, this.sensorEnrichmentConfig).subscribe( + this.sensorEnrichmentConfigService.post(this.sensorName, this.sensorEnrichmentConfig).subscribe( (sensorEnrichmentConfig: SensorEnrichmentConfig) => { }, (error: RestError) => { let msg = ' Sensor parser config but unable to save enrichment configuration: '; this.metronAlerts.showErrorMessage(this.getMessagePrefix() + msg + error.message); }); - this.sensorIndexingConfigService.post(sensorParserConfig.sensorTopic, this.indexingConfigurations).subscribe( + this.sensorIndexingConfigService.post(this.sensorName, this.indexingConfigurations).subscribe( (indexingConfigurations: IndexingConfigurations) => { }, (error: RestError) => { let msg = ' Sensor parser config but unable to save indexing configuration: '; this.metronAlerts.showErrorMessage(this.getMessagePrefix() + msg + error.message); }); - this.metronAlerts.showSuccessMessage(this.getMessagePrefix() + ' Sensor ' + sensorParserConfig.sensorTopic); - this.sensorParserConfigService.dataChangedSource.next([this.sensorParserConfig]); + this.metronAlerts.showSuccessMessage(this.getMessagePrefix() + ' Sensor ' + this.sensorName); + this.sensorParserConfigService.dataChangedSource.next([this.sensorName]); this.goBack(); }, (error: RestError) => { this.metronAlerts.showErrorMessage('Unable to save sensor config: ' + error.message); @@ -385,7 +401,7 @@ export class SensorParserConfigComponent implements OnInit { onShowGrokPane() { if (!this.patternLabel) { - this.patternLabel = this.sensorParserConfig.sensorTopic.toUpperCase(); + this.patternLabel = this.sensorName.toUpperCase(); } this.showPane(this.pane.GROK); } http://git-wip-us.apache.org/repos/asf/metron/blob/6afd95fa/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.html ---------------------------------------------------------------------- diff --git a/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.html b/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.html index bd0b1fd..576b5a3 100644 --- a/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.html +++ b/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.html @@ -40,7 +40,7 @@ <table class="table card-deck" metron-config-table #table (onSort)="onSort($event)"> <thead> <tr> - <th> <metron-config-sorter [sortBy]="'sensorTopic'"> Name </metron-config-sorter> </th> + <th> <metron-config-sorter [sortBy]="'sensorName'"> Name </metron-config-sorter> </th> <th> <metron-config-sorter [sortBy]="'parserClassName'"> Parser </metron-config-sorter> </th> <th> <metron-config-sorter [sortBy]="'status'"> Status </metron-config-sorter> </th> <th> <metron-config-sorter [sortBy]="'latency'"> Latency </metron-config-sorter> </th> @@ -52,8 +52,8 @@ </tr> </thead> <tbody> - <tr *ngFor="let sensor of sensors;" (click)="onSensorRowSelect(sensor.config, $event)" [ngClass]="{'active': (selectedSensors.indexOf(sensor) != -1 || sensorParserConfigService.getSelectedSensor() == sensor.config)}"> - <td>{{ sensor.config.sensorTopic }}</td> + <tr *ngFor="let sensor of sensors;" (click)="onSensorRowSelect(sensor, $event)" [ngClass]="{'active': (selectedSensors.indexOf(sensor) != -1 || selectedSensor == sensor)}"> + <td>{{ sensor.sensorName }}</td> <td>{{ getParserType(sensor.config) }}</td> <td [ngClass]="{'warning-text': (sensor.status == 'Stopped' || sensor.status == 'Disabled')}">{{ sensor.status }}</td> <td>{{ sensor.latency }}</td> @@ -69,11 +69,11 @@ <i data-toggle="tooltip" title="Start parser topology" class="fa fa-play fa-lg" aria-hidden="true" [hidden]="(sensor.status != 'Stopped' || sensor.config['startStopInProgress'])" (click)="onStartSensor(sensor, $event)"></i> <i data-toggle="tooltip" title="Enable parser topology" class="fa fa-check-circle-o fa-lg" aria-hidden="true" [hidden]="(sensor.status != 'Disabled' || sensor.config['startStopInProgress'])" (click)="onEnableSensor(sensor, $event)"></i> - <i data-toggle="tooltip" title="Edit parser topology" class="fa fa-pencil fa-lg" aria-hidden="true" (click)="navigateToSensorEdit(sensor.config, $event)"></i> + <i data-toggle="tooltip" title="Edit parser topology" class="fa fa-pencil fa-lg" aria-hidden="true" (click)="navigateToSensorEdit(sensor, $event)"></i> - <i data-toggle="tooltip" title="Delete parser configuration" class="fa fa-trash-o fa-lg" aria-hidden="true" (click)="deleteSensor($event, [sensor.config])"></i> + <i data-toggle="tooltip" title="Delete parser configuration" class="fa fa-trash-o fa-lg" aria-hidden="true" (click)="deleteSensor($event, [sensor])"></i> </td> - <td><input id="{{ sensor.config.sensorTopic }}" class="fontawesome-checkbox" type="checkbox" name="{{sensor.config.sensorTopic}}" (click)="onRowSelected(sensor, $event)"><label attr.for="{{ sensor.config.sensorTopic }}"></label></td> + <td><input id="{{ sensor.sensorName }}" class="fontawesome-checkbox" type="checkbox" name="{{sensor.sensorName}}" (click)="onRowSelected(sensor, $event)"><label attr.for="{{ sensor.sensorName }}"></label></td> </tr> </tbody> </table> http://git-wip-us.apache.org/repos/asf/metron/blob/6afd95fa/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.spec.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.spec.ts b/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.spec.ts index 5534bea..205d885 100644 --- a/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.spec.ts +++ b/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.spec.ts @@ -77,28 +77,28 @@ class MockSensorParserConfigHistoryService extends SensorParserConfigHistoryServ } class MockSensorParserConfigService extends SensorParserConfigService { - private sensorParserConfigs: SensorParserConfig[]; + private sensorParserConfigs: {}; constructor(private http2: Http, @Inject(APP_CONFIG) private config2: IAppConfig) { super(http2, config2); } - public setSensorParserConfigForTest(sensorParserConfigs: SensorParserConfig[]) { + public setSensorParserConfigForTest(sensorParserConfigs: {}) { this.sensorParserConfigs = sensorParserConfigs; } - public getAll(): Observable<SensorParserConfig[]> { + public getAll(): Observable<{string: SensorParserConfig}> { return Observable.create(observer => { observer.next(this.sensorParserConfigs); observer.complete(); }); } - public deleteSensorParserConfigs(sensors: SensorParserConfig[]): Observable<{success: Array<string>, failure: Array<string>}> { + public deleteSensorParserConfigs(sensorNames: string[]): Observable<{success: Array<string>, failure: Array<string>}> { let result: {success: Array<string>, failure: Array<string>} = {success: [], failure: []}; let observable = Observable.create((observer => { - for (let i = 0; i < sensors.length; i++) { - result.success.push(sensors[i].sensorTopic); + for (let i = 0; i < sensorNames.length; i++) { + result.success.push(sensorNames[i]); } observer.next(result); observer.complete(); @@ -212,8 +212,8 @@ describe('Component: SensorParserList', () => { let sensorParserConfig1 = new SensorParserConfig(); let sensorParserConfig2 = new SensorParserConfig(); - sensorParserConfig1.sensorTopic = 'squid'; - sensorParserConfig2.sensorTopic = 'bro'; + sensorParserConfigHistory1.sensorName = 'squid'; + sensorParserConfigHistory2.sensorName = 'bro'; sensorParserConfigHistory1.config = sensorParserConfig1; sensorParserConfigHistory2.config = sensorParserConfig2; @@ -224,7 +224,7 @@ describe('Component: SensorParserList', () => { sensorParserStatus2.name = 'bro'; sensorParserStatus2.status = 'KILLED'; - sensorParserConfigHistoryService.setSensorParserConfigHistoryForTest([sensorParserConfigHistory1, sensorParserConfigHistory2]); + sensorParserConfigService.setSensorParserConfigForTest({'squid': sensorParserConfig1, 'bro': sensorParserConfig2}); stormService.setTopologyStatusForTest([sensorParserStatus1, sensorParserStatus2]); let component: SensorParserListComponent = fixture.componentInstance; @@ -233,8 +233,8 @@ describe('Component: SensorParserList', () => { component.ngOnInit(); - expect(component.sensors[0]).toEqual(sensorParserConfigHistory1); - expect(component.sensors[1]).toEqual(sensorParserConfigHistory2); + expect(component.sensors[0].sensorName).toEqual(sensorParserConfigHistory1.sensorName); + expect(component.sensors[1].sensorName).toEqual(sensorParserConfigHistory2.sensorName); expect(component.sensorsStatus[0]).toEqual(Object.assign(new TopologyStatus(), sensorParserStatus1)); expect(component.sensorsStatus[1]).toEqual(Object.assign(new TopologyStatus(), sensorParserStatus2)); expect(component.selectedSensors).toEqual([]); @@ -270,9 +270,9 @@ describe('Component: SensorParserList', () => { let component: SensorParserListComponent = fixture.componentInstance; - let sensorParserConfig1 = new SensorParserConfig(); - sensorParserConfig1.sensorTopic = 'squid'; - component.navigateToSensorEdit(sensorParserConfig1, event); + let sensorParserConfigHistory1 = new SensorParserConfigHistory(); + sensorParserConfigHistory1.sensorName = 'squid'; + component.navigateToSensorEdit(sensorParserConfigHistory1, event); let expectStr = router.navigateByUrl['calls'].argsFor(0); expect(expectStr).toEqual(['/sensors(dialog:sensors-config/squid)']); @@ -353,27 +353,27 @@ describe('Component: SensorParserList', () => { it('onSensorRowSelect should change the url and updated the selected items stack', async(() => { - let sensorParserConfig1 = new SensorParserConfig(); - sensorParserConfig1.sensorTopic = 'squid'; + let sensorParserConfigHistory1 = new SensorParserConfigHistory(); + sensorParserConfigHistory1.sensorName = 'squid'; let component: SensorParserListComponent = fixture.componentInstance; let event = {target: {type: 'div', parentElement: {firstChild: {type: 'div'}}}}; - sensorParserConfigService.setSeletedSensor(sensorParserConfig1); - component.onSensorRowSelect(sensorParserConfig1, event); + component.selectedSensor = sensorParserConfigHistory1; + component.onSensorRowSelect(sensorParserConfigHistory1, event); - expect(sensorParserConfigService.getSelectedSensor()).toEqual(null); + expect(component.selectedSensor).toEqual(null); - component.onSensorRowSelect(sensorParserConfig1, event); + component.onSensorRowSelect(sensorParserConfigHistory1, event); - expect(sensorParserConfigService.getSelectedSensor()).toEqual(sensorParserConfig1); + expect(component.selectedSensor).toEqual(sensorParserConfigHistory1); - sensorParserConfigService.setSeletedSensor(sensorParserConfig1); + component.selectedSensor = sensorParserConfigHistory1; event = {target: {type: 'checkbox', parentElement: {firstChild: {type: 'div'}}}}; - component.onSensorRowSelect(sensorParserConfig1, event); + component.onSensorRowSelect(sensorParserConfigHistory1, event); - expect(sensorParserConfigService.getSelectedSensor()).toEqual(sensorParserConfig1); + expect(component.selectedSensor).toEqual(sensorParserConfigHistory1); fixture.destroy(); })); @@ -409,8 +409,8 @@ describe('Component: SensorParserList', () => { let sensorParserConfig1 = new SensorParserConfig(); let sensorParserConfig2 = new SensorParserConfig(); - sensorParserConfig1.sensorTopic = 'squid'; - sensorParserConfig2.sensorTopic = 'bro'; + sensorParserConfigHistory1.sensorName = 'squid'; + sensorParserConfigHistory2.sensorName = 'bro'; sensorParserConfigHistory1.config = sensorParserConfig1; sensorParserConfigHistory2.config = sensorParserConfig2; @@ -421,7 +421,7 @@ describe('Component: SensorParserList', () => { expect(metronAlerts.showSuccessMessage).toHaveBeenCalled(); - component.deleteSensor(event, [sensorParserConfigHistory1.config]); + component.deleteSensor(event, [sensorParserConfigHistory1]); expect(metronDialog.showConfirmationMessage).toHaveBeenCalled(); expect(metronDialog.showConfirmationMessage['calls'].count()).toEqual(2); @@ -627,6 +627,7 @@ describe('Component: SensorParserList', () => { component.sensors = [ Object.assign(new SensorParserConfigHistory(), { + 'sensorName': 'abc', 'config': { 'parserClassName': 'org.apache.metron.parsers.GrokParser', 'sensorTopic': 'abc', @@ -637,6 +638,7 @@ describe('Component: SensorParserList', () => { 'modifiedByDate': '2016-11-25 09:09:12' }), Object.assign(new SensorParserConfigHistory(), { + 'sensorName': 'plm', 'config': { 'parserClassName': 'org.apache.metron.parsers.Bro', 'sensorTopic': 'plm', @@ -647,6 +649,7 @@ describe('Component: SensorParserList', () => { 'modifiedByDate': '2016-11-25 12:39:21' }), Object.assign(new SensorParserConfigHistory(), { + 'sensorName': 'xyz', 'config': { 'parserClassName': 'org.apache.metron.parsers.GrokParser', 'sensorTopic': 'xyz', @@ -658,15 +661,15 @@ describe('Component: SensorParserList', () => { }) ]; - component.onSort({sortBy: 'sensorTopic', sortOrder: Sort.ASC}); - expect(component.sensors[0].config.sensorTopic).toEqual('abc'); - expect(component.sensors[1].config.sensorTopic).toEqual('plm'); - expect(component.sensors[2].config.sensorTopic).toEqual('xyz'); + component.onSort({sortBy: 'sensorName', sortOrder: Sort.ASC}); + expect(component.sensors[0].sensorName).toEqual('abc'); + expect(component.sensors[1].sensorName).toEqual('plm'); + expect(component.sensors[2].sensorName).toEqual('xyz'); - component.onSort({sortBy: 'sensorTopic', sortOrder: Sort.DSC}); - expect(component.sensors[0].config.sensorTopic).toEqual('xyz'); - expect(component.sensors[1].config.sensorTopic).toEqual('plm'); - expect(component.sensors[2].config.sensorTopic).toEqual('abc'); + component.onSort({sortBy: 'sensorName', sortOrder: Sort.DSC}); + expect(component.sensors[0].sensorName).toEqual('xyz'); + expect(component.sensors[1].sensorName).toEqual('plm'); + expect(component.sensors[2].sensorName).toEqual('abc'); component.onSort({sortBy: 'parserClassName', sortOrder: Sort.ASC}); expect(component.sensors[0].config.parserClassName).toEqual('org.apache.metron.parsers.Bro'); @@ -695,6 +698,7 @@ describe('Component: SensorParserList', () => { component.sensors = [ Object.assign(new SensorParserConfigHistory(), { + 'sensorName': 'abc', 'config': { 'parserClassName': 'org.apache.metron.parsers.GrokParser', 'sensorTopic': 'abc', http://git-wip-us.apache.org/repos/asf/metron/blob/6afd95fa/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.ts b/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.ts index ccc8aca..1129914 100644 --- a/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.ts +++ b/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.ts @@ -41,6 +41,7 @@ export class SensorParserListComponent implements OnInit { count: number = 0; sensors: SensorParserConfigHistory[] = []; sensorsStatus: TopologyStatus[] = []; + selectedSensor: SensorParserConfigHistory; selectedSensors: SensorParserConfigHistory[] = []; enableAutoRefresh: boolean = true; @@ -58,9 +59,15 @@ export class SensorParserListComponent implements OnInit { } getSensors(justOnce: boolean) { - this.sensorParserConfigHistoryService.getAll().subscribe( - (results: SensorParserConfigHistory[]) => { - this.sensors = results; + this.sensorParserConfigService.getAll().subscribe( + (results: {string: SensorParserConfig}) => { + this.sensors = []; + for (let sensorName of Object.keys(results)) { + let sensorParserConfigHistory = new SensorParserConfigHistory(); + sensorParserConfigHistory.sensorName = sensorName; + sensorParserConfigHistory.config = results[sensorName]; + this.sensors.push(sensorParserConfigHistory); + } this.selectedSensors = []; this.count = this.sensors.length; if (!justOnce) { @@ -76,12 +83,12 @@ export class SensorParserListComponent implements OnInit { onSort($event: SortEvent) { switch ($event.sortBy) { - case 'sensorTopic': + case 'sensorName': this.sensors.sort((obj1: SensorParserConfigHistory, obj2: SensorParserConfigHistory) => { if ($event.sortOrder === Sort.ASC) { - return obj1.config[$event.sortBy].localeCompare(obj2.config[$event.sortBy]); + return obj1.sensorName.localeCompare(obj2.sensorName); } - return obj2.config[$event.sortBy].localeCompare(obj1.config[$event.sortBy]); + return obj2.sensorName.localeCompare(obj1.sensorName); }); break; case 'parserClassName': @@ -139,7 +146,7 @@ export class SensorParserListComponent implements OnInit { for (let sensor of this.sensors) { let status: TopologyStatus = this.sensorsStatus.find(status => { - return status.name === sensor.config.sensorTopic; + return status.name === sensor.sensorName; }); if (status) { @@ -179,9 +186,9 @@ export class SensorParserListComponent implements OnInit { this.router.navigateByUrl('/sensors(dialog:sensors-config/new)'); } - navigateToSensorEdit(selectedSensor: SensorParserConfig, event) { - this.sensorParserConfigService.setSeletedSensor(selectedSensor); - this.router.navigateByUrl('/sensors(dialog:sensors-config/' + selectedSensor.sensorTopic + ')'); + navigateToSensorEdit(selectedSensor: SensorParserConfigHistory, event) { + this.selectedSensor = selectedSensor; + this.router.navigateByUrl('/sensors(dialog:sensors-config/' + selectedSensor.sensorName + ')'); event.stopPropagation(); } @@ -207,31 +214,30 @@ export class SensorParserListComponent implements OnInit { } } - onSensorRowSelect(sensor: SensorParserConfig, $event) { + onSensorRowSelect(sensor: SensorParserConfigHistory, $event) { if ($event.target.type !== 'checkbox' && $event.target.parentElement.firstChild.type !== 'checkbox') { - if (this.sensorParserConfigService.getSelectedSensor() === sensor) { - this.sensorParserConfigService.setSeletedSensor(null); + if (this.selectedSensor === sensor) { + this.selectedSensor = null; this.router.navigateByUrl('/sensors'); return; } - - this.sensorParserConfigService.setSeletedSensor(sensor); - this.router.navigateByUrl('/sensors(dialog:sensors-readonly/' + sensor.sensorTopic + ')'); + this.selectedSensor = sensor; + this.router.navigateByUrl('/sensors(dialog:sensors-readonly/' + sensor.sensorName + ')'); } } - deleteSensor($event, selectedSensorsToDelete: SensorParserConfig[]) { + deleteSensor($event, selectedSensorsToDelete: SensorParserConfigHistory[]) { if ($event) { $event.stopPropagation(); } - let sensorNames = selectedSensorsToDelete.map(sensor => { return sensor.sensorTopic; }); + let sensorNames = selectedSensorsToDelete.map(sensor => { return sensor.sensorName; }); let confirmationsMsg = 'Are you sure you want to delete sensor(s) ' + sensorNames.join(', ') + ' ?'; this.metronDialogBox.showConfirmationMessage(confirmationsMsg).subscribe(result => { if (result) { - this.sensorParserConfigService.deleteSensorParserConfigs(selectedSensorsToDelete) + this.sensorParserConfigService.deleteSensorParserConfigs(sensorNames) .subscribe((deleteResult: {success: Array<string>, failure: Array<string>}) => { if (deleteResult.success.length > 0) { this.metronAlerts.showSuccessMessage('Deleted sensors: ' + deleteResult.success.join(', ')); @@ -246,10 +252,7 @@ export class SensorParserListComponent implements OnInit { } onDeleteSensor() { - let selectedSensorsToDelete = this.selectedSensors.map(info => { - return info.config; - }); - this.deleteSensor(null, selectedSensorsToDelete); + this.deleteSensor(null, this.selectedSensors); } onStopSensors() { @@ -263,12 +266,12 @@ export class SensorParserListComponent implements OnInit { onStopSensor(sensor: SensorParserConfigHistory, event) { this.toggleStartStopInProgress(sensor); - this.stormService.stopParser(sensor.config.sensorTopic).subscribe(result => { - this.metronAlerts.showSuccessMessage('Stopped sensor ' + sensor.config.sensorTopic); + this.stormService.stopParser(sensor.sensorName).subscribe(result => { + this.metronAlerts.showSuccessMessage('Stopped sensor ' + sensor.sensorName); this.toggleStartStopInProgress(sensor); }, error => { - this.metronAlerts.showErrorMessage('Unable to stop sensor ' + sensor.config.sensorTopic); + this.metronAlerts.showErrorMessage('Unable to stop sensor ' + sensor.sensorName); this.toggleStartStopInProgress(sensor); }); @@ -288,17 +291,17 @@ export class SensorParserListComponent implements OnInit { onStartSensor(sensor: SensorParserConfigHistory, event) { this.toggleStartStopInProgress(sensor); - this.stormService.startParser(sensor.config.sensorTopic).subscribe(result => { + this.stormService.startParser(sensor.sensorName).subscribe(result => { if (result['status'] === 'ERROR') { - this.metronAlerts.showErrorMessage('Unable to start sensor ' + sensor.config.sensorTopic + ': ' + result['message']); + this.metronAlerts.showErrorMessage('Unable to start sensor ' + sensor.sensorName + ': ' + result['message']); } else { - this.metronAlerts.showSuccessMessage('Started sensor ' + sensor.config.sensorTopic); + this.metronAlerts.showSuccessMessage('Started sensor ' + sensor.sensorName); } this.toggleStartStopInProgress(sensor); }, error => { - this.metronAlerts.showErrorMessage('Unable to start sensor ' + sensor.config.sensorTopic); + this.metronAlerts.showErrorMessage('Unable to start sensor ' + sensor.sensorName); this.toggleStartStopInProgress(sensor); }); @@ -318,12 +321,12 @@ export class SensorParserListComponent implements OnInit { onDisableSensor(sensor: SensorParserConfigHistory, event) { this.toggleStartStopInProgress(sensor); - this.stormService.deactivateParser(sensor.config.sensorTopic).subscribe(result => { - this.metronAlerts.showSuccessMessage('Disabled sensor ' + sensor.config.sensorTopic); + this.stormService.deactivateParser(sensor.sensorName).subscribe(result => { + this.metronAlerts.showSuccessMessage('Disabled sensor ' + sensor.sensorName); this.toggleStartStopInProgress(sensor); }, error => { - this.metronAlerts.showErrorMessage('Unable to disable sensor ' + sensor.config.sensorTopic); + this.metronAlerts.showErrorMessage('Unable to disable sensor ' + sensor.sensorName); this.toggleStartStopInProgress(sensor); }); @@ -343,12 +346,12 @@ export class SensorParserListComponent implements OnInit { onEnableSensor(sensor: SensorParserConfigHistory, event) { this.toggleStartStopInProgress(sensor); - this.stormService.activateParser(sensor.config.sensorTopic).subscribe(result => { - this.metronAlerts.showSuccessMessage('Enabled sensor ' + sensor.config.sensorTopic); + this.stormService.activateParser(sensor.sensorName).subscribe(result => { + this.metronAlerts.showSuccessMessage('Enabled sensor ' + sensor.sensorName); this.toggleStartStopInProgress(sensor); }, error => { - this.metronAlerts.showErrorMessage('Unable to enabled sensor ' + sensor.config.sensorTopic); + this.metronAlerts.showErrorMessage('Unable to enabled sensor ' + sensor.sensorName); this.toggleStartStopInProgress(sensor); }); @@ -362,7 +365,7 @@ export class SensorParserListComponent implements OnInit { } onNavigationStart() { - this.sensorParserConfigService.setSeletedSensor(null); + this.selectedSensor = null; this.selectedSensors = []; } } http://git-wip-us.apache.org/repos/asf/metron/blob/6afd95fa/metron-interface/metron-config/src/app/service/sensor-parser-config.service.spec.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-config/src/app/service/sensor-parser-config.service.spec.ts b/metron-interface/metron-config/src/app/service/sensor-parser-config.service.spec.ts index bae4656..c42f127 100644 --- a/metron-interface/metron-config/src/app/service/sensor-parser-config.service.spec.ts +++ b/metron-interface/metron-config/src/app/service/sensor-parser-config.service.spec.ts @@ -93,7 +93,7 @@ describe('SensorParserConfigService', () => { it('post', async(inject([], () => { mockBackend.connections.subscribe((c: MockConnection) => c.mockRespond(sensorParserConfigResponse)); - sensorParserConfigService.post(sensorParserConfig).subscribe( + sensorParserConfigService.post('bro', sensorParserConfig).subscribe( result => { expect(result).toEqual(sensorParserConfig); }, error => console.log(error)); @@ -138,7 +138,7 @@ describe('SensorParserConfigService', () => { it('deleteSensorParserConfigs', async(inject([], () => { mockBackend.connections.subscribe((c: MockConnection) => c.mockRespond(deleteResponse)); - sensorParserConfigService.deleteSensorParserConfigs([sensorParserConfig1, sensorParserConfig2]).subscribe(result => { + sensorParserConfigService.deleteSensorParserConfigs(['bro1', 'bro2']).subscribe(result => { expect(result.success.length).toEqual(2); }); }))); http://git-wip-us.apache.org/repos/asf/metron/blob/6afd95fa/metron-interface/metron-config/src/app/service/sensor-parser-config.service.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-config/src/app/service/sensor-parser-config.service.ts b/metron-interface/metron-config/src/app/service/sensor-parser-config.service.ts index 25cd833..7f1afa2 100644 --- a/metron-interface/metron-config/src/app/service/sensor-parser-config.service.ts +++ b/metron-interface/metron-config/src/app/service/sensor-parser-config.service.ts @@ -31,15 +31,16 @@ export class SensorParserConfigService { defaultHeaders = {'Content-Type': 'application/json', 'X-Requested-With': 'XMLHttpRequest'}; selectedSensorParserConfig: SensorParserConfig; - dataChangedSource = new Subject<SensorParserConfig[]>(); + dataChangedSource = new Subject<string[]>(); dataChanged$ = this.dataChangedSource.asObservable(); constructor(private http: Http, @Inject(APP_CONFIG) private config: IAppConfig) { } - public post(sensorParserConfig: SensorParserConfig): Observable<SensorParserConfig> { - return this.http.post(this.url, JSON.stringify(sensorParserConfig), new RequestOptions({headers: new Headers(this.defaultHeaders)})) + public post(name: string, sensorParserConfig: SensorParserConfig): Observable<SensorParserConfig> { + return this.http.post(this.url + '/' + name, JSON.stringify(sensorParserConfig), + new RequestOptions({headers: new Headers(this.defaultHeaders)})) .map(HttpUtil.extractData) .catch(HttpUtil.handleError); } @@ -50,7 +51,7 @@ export class SensorParserConfigService { .catch(HttpUtil.handleError); } - public getAll(): Observable<SensorParserConfig[]> { + public getAll(): Observable<{}> { return this.http.get(this.url, new RequestOptions({headers: new Headers(this.defaultHeaders)})) .map(HttpUtil.extractData) .catch(HttpUtil.handleError); @@ -73,7 +74,7 @@ export class SensorParserConfigService { .catch(HttpUtil.handleError); } - public deleteSensorParserConfigs(sensors: SensorParserConfig[]): Observable<{success: Array<string>, failure: Array<string>}> { + public deleteSensorParserConfigs(sensorNames: string[]): Observable<{success: Array<string>, failure: Array<string>}> { let result: {success: Array<string>, failure: Array<string>} = {success: [], failure: []}; let observable = Observable.create((observer => { @@ -83,18 +84,17 @@ export class SensorParserConfigService { observer.complete(); } - this.dataChangedSource.next(sensors); + this.dataChangedSource.next(sensorNames); }; - - for (let i = 0; i < sensors.length; i++) { - this.deleteSensorParserConfig(sensors[i].sensorTopic).subscribe(results => { - result.success.push(sensors[i].sensorTopic); - if (result.success.length + result.failure.length === sensors.length) { + for (let i = 0; i < sensorNames.length; i++) { + this.deleteSensorParserConfig(sensorNames[i]).subscribe(results => { + result.success.push(sensorNames[i]); + if (result.success.length + result.failure.length === sensorNames.length) { completed(); } }, error => { - result.failure.push(sensors[i].sensorTopic); - if (result.success.length + result.failure.length === sensors.length) { + result.failure.push(sensorNames[i]); + if (result.success.length + result.failure.length === sensorNames.length) { completed(); } }); @@ -105,12 +105,4 @@ export class SensorParserConfigService { return observable; } - public setSeletedSensor(sensor: SensorParserConfig): void { - this.selectedSensorParserConfig = sensor; - } - - public getSelectedSensor(): SensorParserConfig { - return this.selectedSensorParserConfig; - } - } http://git-wip-us.apache.org/repos/asf/metron/blob/6afd95fa/metron-interface/metron-rest/README.md ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md index c9684a5..02cd946 100644 --- a/metron-interface/metron-rest/README.md +++ b/metron-interface/metron-rest/README.md @@ -581,10 +581,11 @@ Request and Response objects are JSON formatted. The JSON schemas are available * 200 - Returns SensorIndexingConfig * 404 - SensorIndexingConfig is missing -### `POST /api/v1/sensor/parser/config` +### `POST /api/v1/sensor/parser/config/{name}` * Description: Updates or creates a SensorParserConfig in Zookeeper * Input: * sensorParserConfig - SensorParserConfig + * name - SensorEnrichmentConfig name * Returns: * 200 - SensorParserConfig updated. Returns saved SensorParserConfig * 201 - SensorParserConfig created. Returns saved SensorParserConfig http://git-wip-us.apache.org/repos/asf/metron/blob/6afd95fa/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorParserConfigController.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorParserConfigController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorParserConfigController.java index 189e2af..ecbd034 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorParserConfigController.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorParserConfigController.java @@ -47,13 +47,13 @@ public class SensorParserConfigController { @ApiOperation(value = "Updates or creates a SensorParserConfig in Zookeeper") @ApiResponses(value = { @ApiResponse(message = "SensorParserConfig updated. Returns saved SensorParserConfig", code = 200), @ApiResponse(message = "SensorParserConfig created. Returns saved SensorParserConfig", code = 201) }) - @RequestMapping(method = RequestMethod.POST) - ResponseEntity<SensorParserConfig> save(@ApiParam(name="sensorParserConfig", value="SensorParserConfig", required=true)@RequestBody SensorParserConfig sensorParserConfig) throws RestException { - String name = sensorParserConfig.getSensorTopic(); + @RequestMapping(value = "/{name}", method = RequestMethod.POST) + ResponseEntity<SensorParserConfig> save(@ApiParam(name="name", value="SensorParserConfig name", required=true)@PathVariable String name, + @ApiParam(name="sensorParserConfig", value="SensorParserConfig", required=true)@RequestBody SensorParserConfig sensorParserConfig) throws RestException { if (sensorParserConfigService.findOne(name) == null) { - return new ResponseEntity<>(sensorParserConfigService.save(sensorParserConfig), HttpStatus.CREATED); + return new ResponseEntity<>(sensorParserConfigService.save(name, sensorParserConfig), HttpStatus.CREATED); } else { - return new ResponseEntity<>(sensorParserConfigService.save(sensorParserConfig), HttpStatus.OK); + return new ResponseEntity<>(sensorParserConfigService.save(name, sensorParserConfig), HttpStatus.OK); } } @@ -73,7 +73,7 @@ public class SensorParserConfigController { @ApiOperation(value = "Retrieves all SensorParserConfigs from Zookeeper") @ApiResponse(message = "Returns all SensorParserConfigs", code = 200) @RequestMapping(method = RequestMethod.GET) - ResponseEntity<Iterable<SensorParserConfig>> findAll() throws RestException { + ResponseEntity<Map<String, SensorParserConfig>> findAll() throws RestException { return new ResponseEntity<>(sensorParserConfigService.getAll(), HttpStatus.OK); } http://git-wip-us.apache.org/repos/asf/metron/blob/6afd95fa/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java index 9b863b8..2389e09 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java @@ -27,11 +27,11 @@ import java.util.Map; public interface SensorParserConfigService { - SensorParserConfig save(SensorParserConfig sensorParserConfig) throws RestException; + SensorParserConfig save(String name, SensorParserConfig sensorParserConfig) throws RestException; SensorParserConfig findOne(String name) throws RestException; - Iterable<SensorParserConfig> getAll() throws RestException; + Map<String, SensorParserConfig> getAll() throws RestException; List<String> getAllTypes() throws RestException; http://git-wip-us.apache.org/repos/asf/metron/blob/6afd95fa/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java index 7e70344..c460e3c 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java @@ -69,9 +69,9 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService @Override - public SensorParserConfig save(SensorParserConfig sensorParserConfig) throws RestException { + public SensorParserConfig save(String name, SensorParserConfig sensorParserConfig) throws RestException { try { - ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorParserConfig.getSensorTopic(), + ConfigurationsUtils.writeSensorParserConfigToZookeeper(name, objectMapper.writeValueAsString(sensorParserConfig).getBytes(), client); } catch (Exception e) { throw new RestException(e); @@ -86,13 +86,13 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService } @Override - public Iterable<SensorParserConfig> getAll() throws RestException { - List<SensorParserConfig> sensorParserConfigs = new ArrayList<>(); + public Map<String, SensorParserConfig> getAll() throws RestException { + Map<String, SensorParserConfig> sensorParserConfigs = new HashMap<>(); List<String> sensorNames = getAllTypes(); for (String name : sensorNames) { SensorParserConfig config = findOne(name); if(config != null) { - sensorParserConfigs.add(config); + sensorParserConfigs.put(name, config); } } return sensorParserConfigs; http://git-wip-us.apache.org/repos/asf/metron/blob/6afd95fa/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java index 88e5355..c9adac9 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java @@ -208,7 +208,7 @@ public class SensorParserConfigControllerIntegrationTest { numFields.set(numFields.get() + 1); } } - this.mockMvc.perform(post(sensorParserConfigUrl).with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(squidJson)) + this.mockMvc.perform(post(sensorParserConfigUrl + "/squidTest").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(squidJson)) .andExpect(status().isCreated()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$.*", hasSize(numFields.get()))) @@ -242,18 +242,19 @@ public class SensorParserConfigControllerIntegrationTest { this.mockMvc.perform(get(sensorParserConfigUrl).with(httpBasic(user,password))) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) - .andExpect(jsonPath("$[?(@.parserClassName == 'org.apache.metron.parsers.GrokParser' &&" + - "@.sensorTopic == 'squidTest' &&" + - "@.parserConfig.grokPath == 'target/patterns/squidTest' &&" + - "@.parserConfig.patternLabel == 'SQUIDTEST' &&" + - "@.parserConfig.timestampField == 'timestamp' &&" + - "@.fieldTransformations[0].transformation == 'STELLAR' &&" + - "@.fieldTransformations[0].output[0] == 'full_hostname' &&" + - "@.fieldTransformations[0].output[1] == 'domain_without_subdomains' &&" + - "@.fieldTransformations[0].config.full_hostname == 'URL_TO_HOST(url)' &&" + - "@.fieldTransformations[0].config.domain_without_subdomains == 'DOMAIN_REMOVE_SUBDOMAINS(full_hostname)')]").exists()); - - this.mockMvc.perform(post(sensorParserConfigUrl).with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson)) + .andExpect(jsonPath("$.squidTest.*", hasSize(numFields.get()))) + .andExpect(jsonPath("$.squidTest.parserClassName").value("org.apache.metron.parsers.GrokParser")) + .andExpect(jsonPath("$.squidTest.sensorTopic").value("squidTest")) + .andExpect(jsonPath("$.squidTest.parserConfig.grokPath").value("target/patterns/squidTest")) + .andExpect(jsonPath("$.squidTest.parserConfig.patternLabel").value("SQUIDTEST")) + .andExpect(jsonPath("$.squidTest.parserConfig.timestampField").value("timestamp")) + .andExpect(jsonPath("$.squidTest.fieldTransformations[0].transformation").value("STELLAR")) + .andExpect(jsonPath("$.squidTest.fieldTransformations[0].output[0]").value("full_hostname")) + .andExpect(jsonPath("$.squidTest.fieldTransformations[0].output[1]").value("domain_without_subdomains")) + .andExpect(jsonPath("$.squidTest.fieldTransformations[0].config.full_hostname").value("URL_TO_HOST(url)")) + .andExpect(jsonPath("$.squidTest.fieldTransformations[0].config.domain_without_subdomains").value("DOMAIN_REMOVE_SUBDOMAINS(full_hostname)")); + + this.mockMvc.perform(post(sensorParserConfigUrl + "/broTest").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson)) .andExpect(status().isCreated()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$.*", hasSize(numFields.get()))) @@ -263,7 +264,7 @@ public class SensorParserConfigControllerIntegrationTest { .andExpect(jsonPath("$.mergeMetadata").value("true")) .andExpect(jsonPath("$.parserConfig").isEmpty()); - assertEventually(() -> this.mockMvc.perform(post(sensorParserConfigUrl).with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson)) + assertEventually(() -> this.mockMvc.perform(post(sensorParserConfigUrl + "/broTest").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson)) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$.*", hasSize(numFields.get()))) @@ -276,18 +277,24 @@ public class SensorParserConfigControllerIntegrationTest { this.mockMvc.perform(get(sensorParserConfigUrl).with(httpBasic(user,password))) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) - .andExpect(jsonPath("$[?(@.parserClassName == 'org.apache.metron.parsers.GrokParser' &&" + - "@.sensorTopic == 'squidTest' &&" + - "@.parserConfig.grokPath == 'target/patterns/squidTest' &&" + - "@.parserConfig.patternLabel == 'SQUIDTEST' &&" + - "@.parserConfig.timestampField == 'timestamp' &&" + - "@.fieldTransformations[0].transformation == 'STELLAR' &&" + - "@.fieldTransformations[0].output[0] == 'full_hostname' &&" + - "@.fieldTransformations[0].output[1] == 'domain_without_subdomains' &&" + - "@.fieldTransformations[0].config.full_hostname == 'URL_TO_HOST(url)' &&" + - "@.fieldTransformations[0].config.domain_without_subdomains == 'DOMAIN_REMOVE_SUBDOMAINS(full_hostname)')]").exists()) - .andExpect(jsonPath("$[?(@.parserClassName == 'org.apache.metron.parsers.bro.BasicBroParser' && " + - "@.sensorTopic == 'broTest')]").exists()); + .andExpect(jsonPath("$.*", hasSize(2))) + .andExpect(jsonPath("$.squidTest.*", hasSize(numFields.get()))) + .andExpect(jsonPath("$.squidTest.parserClassName").value("org.apache.metron.parsers.GrokParser")) + .andExpect(jsonPath("$.squidTest.sensorTopic").value("squidTest")) + .andExpect(jsonPath("$.squidTest.parserConfig.grokPath").value("target/patterns/squidTest")) + .andExpect(jsonPath("$.squidTest.parserConfig.patternLabel").value("SQUIDTEST")) + .andExpect(jsonPath("$.squidTest.parserConfig.timestampField").value("timestamp")) + .andExpect(jsonPath("$.squidTest.fieldTransformations[0].transformation").value("STELLAR")) + .andExpect(jsonPath("$.squidTest.fieldTransformations[0].output[0]").value("full_hostname")) + .andExpect(jsonPath("$.squidTest.fieldTransformations[0].output[1]").value("domain_without_subdomains")) + .andExpect(jsonPath("$.squidTest.fieldTransformations[0].config.full_hostname").value("URL_TO_HOST(url)")) + .andExpect(jsonPath("$.squidTest.fieldTransformations[0].config.domain_without_subdomains").value("DOMAIN_REMOVE_SUBDOMAINS(full_hostname)")) + .andExpect(jsonPath("$.broTest.parserClassName").value("org.apache.metron.parsers.bro.BasicBroParser")) + .andExpect(jsonPath("$.broTest.*", hasSize(numFields.get()))) + .andExpect(jsonPath("$.broTest.sensorTopic").value("broTest")) + .andExpect(jsonPath("$.broTest.readMetadata").value("true")) + .andExpect(jsonPath("$.broTest.mergeMetadata").value("true")) + .andExpect(jsonPath("$.broTest.parserConfig").isEmpty()); this.mockMvc.perform(delete(sensorParserConfigUrl + "/squidTest").with(httpBasic(user,password)).with(csrf())) .andExpect(status().isOk()); @@ -306,8 +313,8 @@ public class SensorParserConfigControllerIntegrationTest { this.mockMvc.perform(get(sensorParserConfigUrl).with(httpBasic(user,password))) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) - .andExpect(jsonPath("$[?(@.sensorTopic == 'squidTest')]").doesNotExist()) - .andExpect(jsonPath("$[?(@.sensorTopic == 'broTest')]").exists()); + .andExpect(jsonPath("$.squidTest").doesNotExist()) + .andExpect(jsonPath("$.broTest").exists()); this.mockMvc.perform(delete(sensorParserConfigUrl + "/broTest").with(httpBasic(user,password)).with(csrf())) .andExpect(status().isOk()); @@ -318,8 +325,8 @@ public class SensorParserConfigControllerIntegrationTest { this.mockMvc.perform(get(sensorParserConfigUrl).with(httpBasic(user,password))) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) - .andExpect(jsonPath("$[?(@.sensorTopic == 'squidTest')]").doesNotExist()) - .andExpect(jsonPath("$[?(@.sensorTopic == 'broTest')]").doesNotExist()); + .andExpect(jsonPath("$.squidTest").doesNotExist()) + .andExpect(jsonPath("$.broTest").doesNotExist()); this.mockMvc.perform(get(sensorParserConfigUrl + "/list/available").with(httpBasic(user,password))) .andExpect(status().isOk()) http://git-wip-us.apache.org/repos/asf/metron/blob/6afd95fa/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java index a04444d..9a6022c 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java @@ -185,7 +185,7 @@ public class StormControllerIntegrationTest { SensorParserConfig sensorParserConfig = new SensorParserConfig(); sensorParserConfig.setParserClassName("org.apache.metron.parsers.bro.BasicBroParser"); sensorParserConfig.setSensorTopic("broTest"); - sensorParserConfigService.save(sensorParserConfig); + sensorParserConfigService.save("broTest", sensorParserConfig); { final SensorParserConfig expectedSensorParserConfig = sensorParserConfig; //we must wait for the config to find its way into the config. http://git-wip-us.apache.org/repos/asf/metron/blob/6afd95fa/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java index 7998c21..e6163c0 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java @@ -202,9 +202,9 @@ public class SensorParserConfigServiceImplTest { when(cache.get( eq(ParserConfigurations.class))) .thenReturn(configs); - assertEquals(new ArrayList() {{ - add(getTestBroSensorParserConfig()); - add(getTestSquidSensorParserConfig()); + assertEquals(new HashMap() {{ + put("bro", getTestBroSensorParserConfig()); + put("squid", getTestSquidSensorParserConfig()); }}, sensorParserConfigService.getAll()); } @@ -219,7 +219,7 @@ public class SensorParserConfigServiceImplTest { final SensorParserConfig sensorParserConfig = new SensorParserConfig(); sensorParserConfig.setSensorTopic("bro"); - sensorParserConfigService.save(sensorParserConfig); + sensorParserConfigService.save("bro", sensorParserConfig); } @Test @@ -232,7 +232,7 @@ public class SensorParserConfigServiceImplTest { when(setDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro", broJson.getBytes())).thenReturn(new Stat()); when(curatorFramework.setData()).thenReturn(setDataBuilder); - assertEquals(getTestBroSensorParserConfig(), sensorParserConfigService.save(sensorParserConfig)); + assertEquals(getTestBroSensorParserConfig(), sensorParserConfigService.save("bro", sensorParserConfig)); verify(setDataBuilder).forPath(eq(ConfigurationType.PARSER.getZookeeperRoot() + "/bro"), eq(broJson.getBytes())); }