METRON-1547 Solr Comment Fields (justinleet) closes apache/metron#1037
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a68d031b Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a68d031b Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a68d031b Branch: refs/heads/feature/METRON-1554-pcap-query-panel Commit: a68d031b01fe677d84abb0d25aca4f2ceaf90c53 Parents: 9348c60 Author: justinleet <justinjl...@gmail.com> Authored: Tue Jun 5 14:59:29 2018 -0400 Committer: leet <l...@apache.org> Committed: Tue Jun 5 14:59:29 2018 -0400 ---------------------------------------------------------------------- .../alert-details/alert-details.component.ts | 25 ++- .../src/app/model/comment-add-remove-request.ts | 25 +++ .../src/app/service/update.service.ts | 27 +++ .../rest/controller/UpdateController.java | 23 +++ .../metron/rest/service/UpdateService.java | 3 + .../rest/service/impl/UpdateServiceImpl.java | 19 +++ .../UpdateControllerIntegrationTest.java | 92 ++++++++-- .../elasticsearch/dao/ElasticsearchDao.java | 21 +++ .../dao/ElasticsearchMetaAlertDao.java | 22 +++ .../dao/ElasticsearchMetaAlertUpdateDao.java | 23 +++ .../dao/ElasticsearchUpdateDao.java | 70 ++++++++ .../dao/ElasticsearchMetaAlertDaoTest.java | 17 ++ .../ElasticsearchUpdateIntegrationTest.java | 3 +- .../apache/metron/indexing/dao/HBaseDao.java | 93 +++++++++- .../apache/metron/indexing/dao/IndexDao.java | 2 + .../metron/indexing/dao/MultiIndexDao.java | 47 ++++++ .../indexing/dao/search/AlertComment.java | 130 ++++++++++++++ .../dao/update/CommentAddRemoveRequest.java | 78 +++++++++ .../metron/indexing/dao/update/Document.java | 14 +- .../metron/indexing/dao/update/PatchUtil.java | 50 ------ .../metron/indexing/dao/update/UpdateDao.java | 33 +++- .../apache/metron/indexing/dao/InMemoryDao.java | 17 ++ .../indexing/dao/InMemoryMetaAlertDao.java | 17 ++ .../indexing/dao/UpdateIntegrationTest.java | 169 ++++++++++++++++++- .../AbstractLuceneMetaAlertUpdateDaoTest.java | 17 ++ .../integration/HBaseDaoIntegrationTest.java | 79 ++++++++- .../src/main/config/schema/bro/schema.xml | 3 + .../src/main/config/schema/snort/schema.xml | 3 + .../src/main/config/schema/yaf/schema.xml | 3 + .../org/apache/metron/solr/dao/SolrDao.java | 41 ++++- .../metron/solr/dao/SolrMetaAlertDao.java | 21 +++ .../metron/solr/dao/SolrMetaAlertUpdateDao.java | 23 +++ .../metron/solr/dao/SolrRetrieveLatestDao.java | 1 + .../apache/metron/solr/dao/SolrSearchDao.java | 17 ++ .../apache/metron/solr/dao/SolrUpdateDao.java | 114 ++++++++++++- .../apache/metron/solr/dao/SolrUtilities.java | 36 +++- .../org/apache/metron/solr/dao/SolrDaoTest.java | 5 +- .../metron/solr/dao/SolrMetaAlertDaoTest.java | 18 +- .../metron/solr/dao/SolrUpdateDaoTest.java | 94 +++++++++-- .../integration/SolrSearchIntegrationTest.java | 6 +- .../integration/SolrUpdateIntegrationTest.java | 24 ++- .../resources/config/test/conf/managed-schema | 3 + 42 files changed, 1400 insertions(+), 128 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts index e1c1685..6a07e08 100644 --- a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts +++ b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts @@ -30,6 +30,7 @@ import {AlertComment} from './alert-comment'; import {AuthenticationService} from '../../service/authentication.service'; import {MetronDialogBox} from '../../shared/metron-dialog-box'; import {META_ALERTS_INDEX, META_ALERTS_SENSOR_TYPE} from '../../utils/constants'; +import {CommentAddRemoveRequest} from "../../model/comment-add-remove-request"; export enum AlertState { NEW, OPEN, ESCALATE, DISMISS, RESOLVE @@ -204,10 +205,15 @@ export class AlertDetailsComponent implements OnInit { } onAddComment() { - let alertComment = new AlertComment(this.alertCommentStr, this.authenticationService.getCurrentUserName(), new Date().getTime()); - let tAlertComments = this.alertCommentsWrapper.map(alertsWrapper => alertsWrapper.alertComment); - tAlertComments.unshift(alertComment); - this.patchAlert(new Patch('add', '/comments', tAlertComments)); + let commentRequest = new CommentAddRemoveRequest(); + commentRequest.guid = this.alertSource.guid; + commentRequest.comment = this.alertCommentStr; + commentRequest.username = this.authenticationService.getCurrentUserName(); + commentRequest.timestamp = new Date().getTime(); + commentRequest.sensorType = this.alertSourceType; + this.updateService.addComment(commentRequest).subscribe( () => { + this.getData(true); + }); } patchAlert(patch: Patch) { @@ -232,8 +238,15 @@ export class AlertDetailsComponent implements OnInit { this.metronDialogBox.showConfirmationMessage(commentText).subscribe(response => { if (response) { - this.alertCommentsWrapper.splice(index, 1); - this.patchAlert(new Patch('add', '/comments', this.alertCommentsWrapper.map(alertsWrapper => alertsWrapper.alertComment))); + let commentRequest = new CommentAddRemoveRequest(); + commentRequest.guid = this.alertSource.guid; + commentRequest.comment = this.alertCommentsWrapper[index].alertComment.comment; + commentRequest.username = this.alertCommentsWrapper[index].alertComment.username; + commentRequest.timestamp = this.alertCommentsWrapper[index].alertComment.timestamp; + commentRequest.sensorType = this.alertSourceType; + this.updateService.removeComment(commentRequest).subscribe( () => { + this.getData(true); + }); } }); } http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-interface/metron-alerts/src/app/model/comment-add-remove-request.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/src/app/model/comment-add-remove-request.ts b/metron-interface/metron-alerts/src/app/model/comment-add-remove-request.ts new file mode 100644 index 0000000..35f5d86 --- /dev/null +++ b/metron-interface/metron-alerts/src/app/model/comment-add-remove-request.ts @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export class CommentAddRemoveRequest { + guid: string; + comment: string; + username: string; + sensorType: string; + timestamp: number; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-interface/metron-alerts/src/app/service/update.service.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/src/app/service/update.service.ts b/metron-interface/metron-alerts/src/app/service/update.service.ts index 24b55f0..42a4944 100644 --- a/metron-interface/metron-alerts/src/app/service/update.service.ts +++ b/metron-interface/metron-alerts/src/app/service/update.service.ts @@ -31,6 +31,7 @@ import {Utils} from '../utils/utils'; import {Patch} from '../model/patch'; import {META_ALERTS_INDEX, META_ALERTS_SENSOR_TYPE} from '../utils/constants'; import { GlobalConfigService } from './global-config.service'; +import {CommentAddRemoveRequest} from "../model/comment-add-remove-request"; @Injectable() export class UpdateService { @@ -40,6 +41,8 @@ export class UpdateService { alertChangedSource = new Subject<PatchRequest>(); alertChanged$ = this.alertChangedSource.asObservable(); sourceType = 'source:type'; + alertCommentChangedSource = new Subject<CommentAddRemoveRequest>(); + alertCommentChanged$ = this.alertCommentChangedSource.asObservable(); constructor(private http: Http, private globalConfigService: GlobalConfigService) { this.globalConfigService.get().subscribe((config: {}) => { @@ -47,6 +50,30 @@ export class UpdateService { }); } + public addComment(commentRequest: CommentAddRemoveRequest, fireChangeListener = true): Observable<{}> { + let url = '/api/v1/update/add/comment'; + return this.http.post(url, commentRequest, new RequestOptions({headers: new Headers(this.defaultHeaders)})) + .catch(HttpUtil.handleError) + .map(result => { + if (fireChangeListener) { + this.alertCommentChangedSource.next(commentRequest); + } + return result; + }); + } + + public removeComment(commentRequest: CommentAddRemoveRequest, fireChangeListener = true): Observable<{}> { + let url = '/api/v1/update/remove/comment'; + return this.http.post(url, commentRequest, new RequestOptions({headers: new Headers(this.defaultHeaders)})) + .catch(HttpUtil.handleError) + .map(result => { + if (fireChangeListener) { + this.alertCommentChangedSource.next(commentRequest); + } + return result; + }); + } + public patch(patchRequest: PatchRequest, fireChangeListener = true): Observable<{}> { let url = '/api/v1/update/patch'; return this.http.patch(url, patchRequest, new RequestOptions({headers: new Headers(this.defaultHeaders)})) http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java index 56b0b7b..609442b 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java @@ -20,6 +20,7 @@ package org.apache.metron.rest.controller; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import io.swagger.annotations.ApiResponse; +import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.OriginalNotFoundException; import org.apache.metron.indexing.dao.update.PatchRequest; import org.apache.metron.indexing.dao.update.ReplaceRequest; @@ -67,4 +68,26 @@ public class UpdateController { service.replace(request); return new ResponseEntity<>(HttpStatus.OK); } + + @ApiOperation(value = "Add a comment to an alert") + @ApiResponse(message = "Nothing", code = 200) + @RequestMapping(value = "/add/comment", method = RequestMethod.POST) + ResponseEntity<Void> addCommentToAlert( + @RequestBody @ApiParam(name = "request", value = "Comment add request", required = true) final + CommentAddRemoveRequest request + ) throws RestException { + service.addComment(request); + return new ResponseEntity<>(HttpStatus.OK); + } + + @ApiOperation(value = "Remove a comment to an alert") + @ApiResponse(message = "Nothing", code = 200) + @RequestMapping(value = "/remove/comment", method = RequestMethod.POST) + ResponseEntity<Void> removeCommentFromAlert( + @RequestBody @ApiParam(name = "request", value = "Comment remove request", required = true) final + CommentAddRemoveRequest request + ) throws RestException { + service.removeComment(request); + return new ResponseEntity<>(HttpStatus.OK); + } } http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java index 4cdf4b3..bd59f39 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java @@ -17,6 +17,7 @@ */ package org.apache.metron.rest.service; +import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.OriginalNotFoundException; import org.apache.metron.indexing.dao.update.PatchRequest; import org.apache.metron.indexing.dao.update.ReplaceRequest; @@ -26,4 +27,6 @@ public interface UpdateService { void patch(PatchRequest request) throws RestException, OriginalNotFoundException; void replace(ReplaceRequest request) throws RestException; + void addComment(CommentAddRemoveRequest request) throws RestException; + void removeComment(CommentAddRemoveRequest request) throws RestException; } http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java index 6a42248..49490fd 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java @@ -18,6 +18,7 @@ package org.apache.metron.rest.service.impl; import org.apache.metron.indexing.dao.IndexDao; +import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.OriginalNotFoundException; import org.apache.metron.indexing.dao.update.PatchRequest; import org.apache.metron.indexing.dao.update.ReplaceRequest; @@ -59,4 +60,22 @@ public class UpdateServiceImpl implements UpdateService { throw new RestException(e.getMessage(), e); } } + + @Override + public void addComment(CommentAddRemoveRequest request) throws RestException { + try { + dao.addCommentToAlert(request); + } catch (Exception e) { + throw new RestException(e.getMessage(), e); + } + } + + @Override + public void removeComment(CommentAddRemoveRequest request) throws RestException { + try { + dao.removeCommentFromAlert(request); + } catch (Exception e) { + throw new RestException(e.getMessage(), e); + } + } } http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java index e437325..6b8d5d3 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java @@ -17,15 +17,29 @@ */ package org.apache.metron.rest.controller; +import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic; +import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.patch; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + import com.google.common.collect.ImmutableMap; +import java.util.NavigableMap; import org.adrianwalker.multilinestring.Multiline; import org.apache.curator.framework.CuratorFramework; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Result; -import org.apache.metron.hbase.mock.MockHTable; +import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.hbase.mock.MockHBaseTableProvider; +import org.apache.metron.hbase.mock.MockHTable; import org.apache.metron.indexing.dao.HBaseDao; import org.apache.metron.indexing.dao.SearchIntegrationTest; +import org.apache.metron.indexing.dao.search.AlertComment; +import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.rest.service.UpdateService; import org.junit.Assert; import org.junit.Before; @@ -37,28 +51,17 @@ import org.springframework.http.MediaType; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.MvcResult; import org.springframework.test.web.servlet.ResultActions; import org.springframework.test.web.servlet.setup.MockMvcBuilders; import org.springframework.web.context.WebApplicationContext; -import java.util.NavigableMap; - -import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; -import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; -import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; -import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.patch; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles(TEST_PROFILE) public class UpdateControllerIntegrationTest extends DaoControllerTest { @Autowired - private UpdateService searchService; + private UpdateService updateService; @Autowired public CuratorFramework client; @@ -115,6 +118,30 @@ public class UpdateControllerIntegrationTest extends DaoControllerTest { @Multiline public static String replace; + /** + { + "guid" : "bro_2", + "sensorType" : "bro", + "comment": "test_comment", + "username" : "test_username", + "timestamp":0 + } + */ + @Multiline + public static String addComment; + + /** + { + "guid" : "bro_2", + "sensorType" : "bro", + "comment": "test_comment", + "username" : "test_username", + "timestamp":0 + } + */ + @Multiline + public static String removeComment; + @Before public void setup() throws Exception { this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build(); @@ -191,4 +218,41 @@ public class UpdateControllerIntegrationTest extends DaoControllerTest { } } + @Test + public void shouldAddComment() throws Exception { + CommentAddRemoveRequest commentAddRemoveRequest = new CommentAddRemoveRequest(); + commentAddRemoveRequest.setGuid("bro_1"); + commentAddRemoveRequest.setSensorType("bro"); + commentAddRemoveRequest.setComment("test_comment"); + commentAddRemoveRequest.setUsername("test_username"); + commentAddRemoveRequest.setTimestamp(0L); + + updateService.addComment(commentAddRemoveRequest); + + ResultActions result = this.mockMvc.perform( + post(updateUrl + "/add/comment") + .with(httpBasic(user, password)).with(csrf()) + .contentType(MediaType.parseMediaType("application/json;charset=UTF-8")) + .content(addComment)); + result.andExpect(status().isOk()); + } + + @Test + public void shouldRemoveComment() throws Exception { + CommentAddRemoveRequest commentAddRemoveRequest = new CommentAddRemoveRequest(); + commentAddRemoveRequest.setGuid("bro_1"); + commentAddRemoveRequest.setSensorType("bro"); + commentAddRemoveRequest.setComment("test_comment"); + commentAddRemoveRequest.setUsername("test_username"); + commentAddRemoveRequest.setTimestamp(0L); + + updateService.removeComment(commentAddRemoveRequest); + + ResultActions result = this.mockMvc.perform( + post(updateUrl + "/remove/comment") + .with(httpBasic(user, password)).with(csrf()) + .contentType(MediaType.parseMediaType("application/json;charset=UTF-8")) + .content(removeComment)); + result.andExpect(status().isOk()); + } } http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java index 246de6a..eae0a39 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java @@ -33,6 +33,7 @@ import org.apache.metron.indexing.dao.search.GroupResponse; import org.apache.metron.indexing.dao.search.InvalidSearchException; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; +import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.update.OriginalNotFoundException; import org.apache.metron.indexing.dao.update.PatchRequest; @@ -150,6 +151,16 @@ public class ElasticsearchDao implements IndexDao { } @Override + public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException { + updateDao.addCommentToAlert(request); + } + + @Override + public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException { + updateDao.removeCommentFromAlert(request); + } + + @Override public Map<String, FieldType> getColumnMetadata(List<String> indices) throws IOException { return this.columnMetadataDao.getColumnMetadata(indices); } @@ -159,6 +170,16 @@ public class ElasticsearchDao implements IndexDao { return retrieveLatestDao.getLatestResult(request); } + @Override + public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException { + this.updateDao.addCommentToAlert(request, latest); + } + + @Override + public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException { + this.updateDao.removeCommentFromAlert(request, latest); + } + protected Optional<String> getIndexName(String guid, String sensorType) { return updateDao.getIndexName(guid, sensorType); } http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java index faec939..ab6c40c 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java @@ -41,6 +41,8 @@ import org.apache.metron.indexing.dao.search.InvalidCreateException; import org.apache.metron.indexing.dao.search.InvalidSearchException; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; +import org.apache.metron.indexing.dao.search.SearchResult; +import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.update.OriginalNotFoundException; import org.apache.metron.indexing.dao.update.PatchRequest; @@ -216,6 +218,26 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao { } @Override + public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException { + indexDao.addCommentToAlert(request); + } + + @Override + public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException { + indexDao.removeCommentFromAlert(request); + } + + @Override + public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException { + indexDao.addCommentToAlert(request, latest); + } + + @Override + public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException { + indexDao.removeCommentFromAlert(request, latest); + } + + @Override public void patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request, Optional<Long> timestamp) throws OriginalNotFoundException, IOException { http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java index 6c709a6..d3bdcbb 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java @@ -44,6 +44,7 @@ import org.apache.metron.indexing.dao.metaalert.lucene.AbstractLuceneMetaAlertUp import org.apache.metron.indexing.dao.search.GetRequest; import org.apache.metron.indexing.dao.search.InvalidCreateException; import org.apache.metron.indexing.dao.search.SearchResponse; +import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.Document; import org.elasticsearch.index.query.InnerHitBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -185,6 +186,28 @@ public class ElasticsearchMetaAlertUpdateDao extends AbstractLuceneMetaAlertUpda } } + @Override + public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException { + getUpdateDao().addCommentToAlert(request); + } + + @Override + public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException { + getUpdateDao().removeCommentFromAlert(request); + } + + @Override + public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) + throws IOException { + getUpdateDao().addCommentToAlert(request, latest); + } + + @Override + public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) + throws IOException { + getUpdateDao().removeCommentFromAlert(request, latest); + } + /** * Given an alert GUID, retrieve all associated meta alerts. * @param alertGuid The GUID of the child alert http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDao.java index c4d7412..f2b08d2 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDao.java @@ -17,14 +17,21 @@ */ package org.apache.metron.elasticsearch.dao; +import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD; + import java.io.IOException; import java.lang.invoke.MethodHandles; +import java.util.ArrayList; import java.util.Arrays; import java.util.Date; +import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; import org.apache.metron.elasticsearch.utils.ElasticsearchUtils; import org.apache.metron.indexing.dao.AccessConfig; +import org.apache.metron.indexing.dao.search.AlertComment; +import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.update.UpdateDao; import org.elasticsearch.action.bulk.BulkRequestBuilder; @@ -103,6 +110,69 @@ public class ElasticsearchUpdateDao implements UpdateDao { } } + @Override + @SuppressWarnings("unchecked") + public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException { + Document latest = retrieveLatestDao.getLatest(request.getGuid(), request.getSensorType()); + addCommentToAlert(request, latest); + } + + @Override + @SuppressWarnings("unchecked") + public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException { + if (latest == null) { + return; + } + List<Map<String, Object>> commentsField = (List<Map<String, Object>>) latest.getDocument() + .getOrDefault(COMMENTS_FIELD, new ArrayList<>()); + List<Map<String, Object>> originalComments = new ArrayList<>(commentsField); + + originalComments.add( + new AlertComment(request.getComment(), request.getUsername(), request.getTimestamp()) + .asMap()); + + Document newVersion = new Document(latest); + newVersion.getDocument().put(COMMENTS_FIELD, originalComments); + update(newVersion, Optional.empty()); + } + + @Override + @SuppressWarnings("unchecked") + public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException { + Document latest = retrieveLatestDao.getLatest(request.getGuid(), request.getSensorType()); + removeCommentFromAlert(request, latest); + } + + @Override + @SuppressWarnings("unchecked") + public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException { + if (latest == null) { + return; + } + List<Map<String, Object>> commentsField = (List<Map<String, Object>>) latest.getDocument() + .getOrDefault(COMMENTS_FIELD, new ArrayList<>()); + List<Map<String, Object>> originalComments = new ArrayList<>(commentsField); + + List<AlertComment> alertComments = new ArrayList<>(); + for (Map<String, Object> commentRaw : originalComments) { + alertComments.add(new AlertComment(commentRaw)); + } + + alertComments.remove( + new AlertComment(request.getComment(), request.getUsername(), request.getTimestamp())); + List<Map<String, Object>> commentsFinal = alertComments.stream().map(AlertComment::asMap) + .collect(Collectors.toList()); + Document newVersion = new Document(latest); + if (commentsFinal.size() > 0) { + newVersion.getDocument().put(COMMENTS_FIELD, commentsFinal); + update(newVersion, Optional.empty()); + } else { + newVersion.getDocument().remove(COMMENTS_FIELD); + } + + update(newVersion, Optional.empty()); + } + protected String getIndexName(Document update, Optional<String> index, String indexPostFix) { return index.orElse(getIndexName(update.getGuid(), update.getSensorType()) .orElse(ElasticsearchUtils.getIndexName(update.getSensorType(), indexPostFix, null)) http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java index 25799ad..a3a5f16 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java @@ -35,6 +35,7 @@ import org.apache.metron.indexing.dao.search.GroupResponse; import org.apache.metron.indexing.dao.search.InvalidCreateException; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; +import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.Document; import org.junit.Test; @@ -81,6 +82,22 @@ public class ElasticsearchMetaAlertDaoTest { public Map<String, FieldType> getColumnMetadata(List<String> indices) { return null; } + + @Override + public void addCommentToAlert(CommentAddRemoveRequest request) { + } + + @Override + public void removeCommentFromAlert(CommentAddRemoveRequest request) { + } + + @Override + public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) { + } + + @Override + public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) { + } }; ElasticsearchMetaAlertDao metaAlertDao = new ElasticsearchMetaAlertDao(); metaAlertDao.init(dao); http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java index 97993ff..c5c0bc1 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java @@ -85,8 +85,9 @@ public class ElasticsearchUpdateIntegrationTest extends UpdateIntegrationTest { globalConfig.put(HBaseDao.HBASE_CF, CF); accessConfig.setGlobalConfigSupplier(() -> globalConfig); - dao = new MultiIndexDao(hbaseDao, createDao()); + MultiIndexDao dao = new MultiIndexDao(hbaseDao, createDao()); dao.init(accessConfig); + setDao(dao); } @After http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java index ebb9907..f22372e 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java @@ -28,8 +28,7 @@ import java.util.List; import java.util.Map; import java.util.NavigableMap; import java.util.Optional; - -import com.google.common.hash.Hasher; +import java.util.stream.Collectors; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HTableInterface; @@ -38,6 +37,7 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.util.Bytes; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.common.utils.KeyUtil; +import org.apache.metron.indexing.dao.search.AlertComment; import org.apache.metron.indexing.dao.search.FieldType; import org.apache.metron.indexing.dao.search.GetRequest; import org.apache.metron.indexing.dao.search.GroupRequest; @@ -45,6 +45,7 @@ import org.apache.metron.indexing.dao.search.GroupResponse; import org.apache.metron.indexing.dao.search.InvalidSearchException; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; +import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.Document; /** @@ -210,7 +211,21 @@ public class HBaseDao implements IndexDao { if(entry.getValue()!= null) { Map<String, Object> json = JSONUtils.INSTANCE.load(new String(entry.getValue()), JSONUtils.MAP_SUPPLIER); + + // Make sure comments are in the proper format + @SuppressWarnings("unchecked") + List<Map<String, Object>> commentsMap = (List<Map<String, Object>>) json.get(COMMENTS_FIELD); try { + if (commentsMap != null) { + List<AlertComment> comments = new ArrayList<>(); + for (Map<String, Object> commentMap : commentsMap) { + comments.add(new AlertComment(commentMap)); + } + if (comments.size() > 0) { + json.put(COMMENTS_FIELD, + comments.stream().map(AlertComment::asMap).collect(Collectors.toList())); + } + } Key k = Key.fromBytes(result.getRow()); return new Document(json, k.getGuid(), k.getSensorType(), ts); } catch (IOException e) { @@ -262,4 +277,78 @@ public class HBaseDao implements IndexDao { public Map<String, FieldType> getColumnMetadata(List<String> indices) throws IOException { return null; } + + @Override + @SuppressWarnings("unchecked") + public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException { + Document latest = getLatest(request.getGuid(), request.getSensorType()); + addCommentToAlert(request, latest); + } + + @Override + @SuppressWarnings("unchecked") + public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException { + if (latest == null || latest.getDocument() == null) { + throw new IOException("Unable to add comment to document that doesn't exist"); + } + + List<Map<String, Object>> comments = (List<Map<String, Object>>) latest.getDocument() + .getOrDefault(COMMENTS_FIELD, new ArrayList<>()); + List<Map<String, Object>> originalComments = new ArrayList<>(comments); + + // Convert all comments back to raw JSON before updating. + List<Map<String, Object>> commentsMap = new ArrayList<>(); + for (Map<String, Object> comment : originalComments) { + commentsMap.add(new AlertComment(comment).asMap()); + } + commentsMap.add(new AlertComment( + request.getComment(), + request.getUsername(), + request.getTimestamp()) + .asMap()); + + Document newVersion = new Document(latest); + newVersion.getDocument().put(COMMENTS_FIELD, commentsMap); + update(newVersion, Optional.empty()); + } + + @Override + @SuppressWarnings("unchecked") + public void removeCommentFromAlert(CommentAddRemoveRequest request) + throws IOException { + Document latest = getLatest(request.getGuid(), request.getSensorType()); + removeCommentFromAlert(request, latest); + } + + @Override + @SuppressWarnings("unchecked") + public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) + throws IOException { + if (latest == null || latest.getDocument() == null) { + throw new IOException("Unable to remove comment document that doesn't exist"); + } + List<Map<String, Object>> commentMap = (List<Map<String, Object>>) latest.getDocument().get(COMMENTS_FIELD); + // Can't remove anything if there's nothing there + if (commentMap == null) { + return; + } + List<Map<String, Object>> originalComments = new ArrayList<>(commentMap); + List<AlertComment> comments = new ArrayList<>(); + for (Map<String, Object> commentStr : originalComments) { + comments.add(new AlertComment(commentStr)); + } + + comments.remove(new AlertComment(request.getComment(), request.getUsername(), request.getTimestamp())); + Document newVersion = new Document(latest); + if (comments.size() > 0) { + List<Map<String, Object>> commentsAsMap = comments.stream().map(AlertComment::asMap) + .collect(Collectors.toList()); + newVersion.getDocument().put(COMMENTS_FIELD, commentsAsMap); + update(newVersion, Optional.empty()); + } else { + newVersion.getDocument().remove(COMMENTS_FIELD); + } + + update(newVersion, Optional.empty()); + } } http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java index 4187428..11b2ff0 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java @@ -26,6 +26,8 @@ import org.apache.metron.indexing.dao.update.UpdateDao; */ public interface IndexDao extends UpdateDao, SearchDao, RetrieveLatestDao, ColumnMetadataDao { + String COMMENTS_FIELD = "comments"; + /** * Initialize the DAO with the AccessConfig object. * @param config The config to use for initialization http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java index dad08d6..420c775 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; @@ -36,6 +37,7 @@ import org.apache.metron.indexing.dao.search.GroupResponse; import org.apache.metron.indexing.dao.search.InvalidSearchException; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; +import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.Document; public class MultiIndexDao implements IndexDao { @@ -98,6 +100,51 @@ public class MultiIndexDao implements IndexDao { return null; } + @Override + public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException { + Document latest = getLatest(request.getGuid(), request.getSensorType()); + addCommentToAlert(request, latest); + } + + + @Override + public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException { + List<String> exceptions = + indices.parallelStream().map(dao -> { + try { + dao.addCommentToAlert(request, latest); + return null; + } catch (Throwable e) { + return dao.getClass() + ": " + e.getMessage() + "\n" + ExceptionUtils.getStackTrace(e); + } + }).filter(Objects::nonNull).collect(Collectors.toList()); + if (exceptions.size() > 0) { + throw new IOException(Joiner.on("\n").join(exceptions)); + } + } + + @Override + public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException { + Document latest = getLatest(request.getGuid(), request.getSensorType()); + removeCommentFromAlert(request, latest); + } + + @Override + public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException { + List<String> exceptions = + indices.parallelStream().map(dao -> { + try { + dao.removeCommentFromAlert(request, latest); + return null; + } catch (Throwable e) { + return dao.getClass() + ": " + e.getMessage() + "\n" + ExceptionUtils.getStackTrace(e); + } + }).filter(Objects::nonNull).collect(Collectors.toList()); + if (exceptions.size() > 0) { + throw new IOException(Joiner.on("\n").join(exceptions)); + } + } + private static class DocumentContainer { private Optional<Document> d = Optional.empty(); private Optional<Throwable> t = Optional.empty(); http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/AlertComment.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/AlertComment.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/AlertComment.java new file mode 100644 index 0000000..04aac60 --- /dev/null +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/AlertComment.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.indexing.dao.search; + +import java.util.HashMap; +import java.util.Map; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; + +public class AlertComment { + + private static final String COMMENT_FIELD = "comment"; + private static final String COMMENT_USERNAME_FIELD = "username"; + private static final String COMMENT_TIMESTAMP_FIELD = "timestamp"; + private String comment; + private String username; + private long timestamp; + + private JSONParser parser = new JSONParser(); + + public AlertComment(String comment, String username, long timestamp) { + this.comment = comment; + this.username = username; + this.timestamp = timestamp; + } + + public AlertComment(String json) throws ParseException { + JSONObject parsed = (JSONObject) parser.parse(json); + this.comment = (String) parsed.get(COMMENT_FIELD); + this.username = (String) parsed.get(COMMENT_USERNAME_FIELD); + this.timestamp = (long) parsed.get(COMMENT_TIMESTAMP_FIELD); + } + + public AlertComment(Map<String, Object> comment) { + this.comment = (String) comment.get(COMMENT_FIELD); + this.username = (String) comment.get(COMMENT_USERNAME_FIELD); + this.timestamp = (long) comment.get(COMMENT_TIMESTAMP_FIELD); + } + + public String getComment() { + return comment; + } + + public String getUsername() { + return username; + } + + public long getTimestamp() { + return timestamp; + } + + @SuppressWarnings("unchecked") + public String asJson() { + return asJSONObject().toJSONString(); + } + + @SuppressWarnings("unchecked") + public Map<String, Object> asMap() { + Map<String, Object> map = new HashMap<>(); + map.put(COMMENT_FIELD, comment); + map.put(COMMENT_USERNAME_FIELD, username); + map.put(COMMENT_TIMESTAMP_FIELD, timestamp); + return map; + } + + @SuppressWarnings("unchecked") + public JSONObject asJSONObject() { + JSONObject json = new JSONObject(); + json.put(COMMENT_FIELD, comment); + json.put(COMMENT_USERNAME_FIELD, username); + json.put(COMMENT_TIMESTAMP_FIELD, timestamp); + return json; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + AlertComment that = (AlertComment) o; + + if (getTimestamp() != that.getTimestamp()) { + return false; + } + if (getComment() != null ? !getComment().equals(that.getComment()) + : that.getComment() != null) { + return false; + } + return getUsername() != null ? getUsername().equals(that.getUsername()) + : that.getUsername() == null; + } + + @Override + public int hashCode() { + int result = getComment() != null ? getComment().hashCode() : 0; + result = 31 * result + (getUsername() != null ? getUsername().hashCode() : 0); + result = 31 * result + (int) (getTimestamp() ^ (getTimestamp() >>> 32)); + return result; + } + + @Override + public String toString() { + return "AlertComment{" + + "comment='" + comment + '\'' + + ", username='" + username + '\'' + + ", timestamp=" + timestamp + + '}'; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/CommentAddRemoveRequest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/CommentAddRemoveRequest.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/CommentAddRemoveRequest.java new file mode 100644 index 0000000..8e8bde7 --- /dev/null +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/CommentAddRemoveRequest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.indexing.dao.update; + +public class CommentAddRemoveRequest { + private String guid; + private String sensorType; + private String comment; + private String username; + private long timestamp; + + public String getGuid() { + return guid; + } + + public void setGuid(String guid) { + this.guid = guid; + } + + public String getSensorType() { + return sensorType; + } + + public void setSensorType(String sensorType) { + this.sensorType = sensorType; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public long getTimestamp() { + return timestamp; + } + + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + + @Override + public String toString() { + return "CommentAddRemoveRequest{" + + "guid='" + guid + '\'' + + ", sensorType='" + sensorType + '\'' + + ", comment='" + comment + '\'' + + ", username='" + username + '\'' + + ", timestamp=" + timestamp + + '}'; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java index 6f2f779..3686b19 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java @@ -18,10 +18,10 @@ package org.apache.metron.indexing.dao.update; -import org.apache.metron.common.utils.JSONUtils; - import java.io.IOException; +import java.util.HashMap; import java.util.Map; +import org.apache.metron.common.utils.JSONUtils; public class Document { Long timestamp; @@ -36,7 +36,6 @@ public class Document { setSensorType(sensorType); } - public Document(String document, String guid, String sensorType, Long timestamp) throws IOException { this(convertDoc(document), guid, sensorType, timestamp); } @@ -45,6 +44,15 @@ public class Document { this( document, guid, sensorType, null); } + /** + * Copy constructor + * @param other The document to be copied. + */ + public Document(Document other) { + this(new HashMap<>(other.getDocument()), other.getGuid(), other.getSensorType(), + other.getTimestamp()); + } + private static Map<String, Object> convertDoc(String document) throws IOException { return JSONUtils.INSTANCE.load(document, JSONUtils.MAP_SUPPLIER); } http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/PatchUtil.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/PatchUtil.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/PatchUtil.java deleted file mode 100644 index 5a4ef27..0000000 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/PatchUtil.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.metron.indexing.dao.update; - -import java.io.IOException; -import java.util.Map; -import java.util.Optional; -import org.apache.metron.common.utils.JSONUtils; -import org.apache.metron.indexing.dao.RetrieveLatestDao; - -public class PatchUtil { - - public static Document getPatchedDocument( - RetrieveLatestDao retrieveLatestDao, - PatchRequest request - , Optional<Long> timestamp - ) throws OriginalNotFoundException, IOException { - Map<String, Object> latest = request.getSource(); - if (latest == null) { - Document latestDoc = retrieveLatestDao.getLatest(request.getGuid(), request.getSensorType()); - if (latestDoc != null && latestDoc.getDocument() != null) { - latest = latestDoc.getDocument(); - } else { - throw new OriginalNotFoundException( - "Unable to patch an document that doesn't exist and isn't specified."); - } - } - Map<String, Object> updated = JSONUtils.INSTANCE.applyPatch(request.getPatch(), latest); - return new Document(updated - , request.getGuid() - , request.getSensorType() - , timestamp.orElse(System.currentTimeMillis())); - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java index 6f136ea..b5f38e4 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java @@ -18,8 +18,10 @@ package org.apache.metron.indexing.dao.update; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.Optional; +import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.indexing.dao.RetrieveLatestDao; public interface UpdateDao { @@ -43,6 +45,15 @@ public interface UpdateDao { */ void batchUpdate(Map<Document, Optional<String>> updates) throws IOException; + void addCommentToAlert(CommentAddRemoveRequest request) throws IOException; + + void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException; + + void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException; + + void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException; + + /** * Update a document in an index given a JSON Patch (see RFC 6902 at * https://tools.ietf.org/html/rfc6902) @@ -54,10 +65,30 @@ public interface UpdateDao { default void patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request , Optional<Long> timestamp ) throws OriginalNotFoundException, IOException { - Document d = PatchUtil.getPatchedDocument(retrieveLatestDao, request, timestamp); + Document d = getPatchedDocument(retrieveLatestDao, request, timestamp); update(d, Optional.ofNullable(request.getIndex())); } + default Document getPatchedDocument(RetrieveLatestDao retrieveLatestDao, PatchRequest request, + Optional<Long> timestamp + ) throws OriginalNotFoundException, IOException { + Map<String, Object> latest = request.getSource(); + if (latest == null) { + Document latestDoc = retrieveLatestDao.getLatest(request.getGuid(), request.getSensorType()); + if (latestDoc != null && latestDoc.getDocument() != null) { + latest = latestDoc.getDocument(); + } else { + throw new OriginalNotFoundException( + "Unable to patch an document that doesn't exist and isn't specified."); + } + } + + Map<String, Object> updated = JSONUtils.INSTANCE.applyPatch(request.getPatch(), latest); + return new Document(updated, + request.getGuid(), + request.getSensorType(), + timestamp.orElse(System.currentTimeMillis())); + } /** * Replace a document in an index. http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java index d6e1521..e306567 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java @@ -35,6 +35,7 @@ import org.apache.metron.indexing.dao.search.SearchResponse; import org.apache.metron.indexing.dao.search.SearchResult; import org.apache.metron.indexing.dao.search.SortField; import org.apache.metron.indexing.dao.search.SortOrder; +import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.Document; import java.io.IOException; @@ -291,6 +292,22 @@ public class InMemoryDao implements IndexDao { return indexColumnMetadata; } + @Override + public void addCommentToAlert(CommentAddRemoveRequest request) { + } + + @Override + public void removeCommentFromAlert(CommentAddRemoveRequest request) { + } + + @Override + public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) { + } + + @Override + public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) { + } + public static void setColumnMetadata(Map<String, Map<String, FieldType>> columnMetadata) { Map<String, Map<String, FieldType>> columnMetadataMap = new HashMap<>(); for (Map.Entry<String, Map<String, FieldType>> e: columnMetadata.entrySet()) { http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java index 803d320..9e95ee9 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java @@ -47,6 +47,7 @@ import org.apache.metron.indexing.dao.search.InvalidSearchException; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; import org.apache.metron.indexing.dao.search.SearchResult; +import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.update.OriginalNotFoundException; import org.apache.metron.indexing.dao.update.PatchRequest; @@ -127,6 +128,22 @@ public class InMemoryMetaAlertDao implements MetaAlertDao { } @Override + public void addCommentToAlert(CommentAddRemoveRequest request) { + } + + @Override + public void removeCommentFromAlert(CommentAddRemoveRequest request) { + } + + @Override + public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) { + } + + @Override + public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) { + } + + @Override public Optional<Map<String, Object>> getLatestResult(GetRequest request) throws IOException { return indexDao.getLatestResult(request); } http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java index eebf0bb..1e35523 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java @@ -14,30 +14,61 @@ */ package org.apache.metron.indexing.dao; +import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD; + +import java.io.IOException; import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.NavigableMap; import java.util.Optional; +import java.util.stream.Collectors; +import org.adrianwalker.multilinestring.Multiline; +import org.apache.commons.collections.MapUtils; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Result; import org.apache.metron.common.Constants; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.hbase.mock.MockHTable; +import org.apache.metron.indexing.dao.search.AlertComment; +import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.Document; +import org.apache.metron.indexing.dao.update.OriginalNotFoundException; +import org.apache.metron.indexing.dao.update.PatchRequest; import org.apache.metron.indexing.dao.update.ReplaceRequest; import org.junit.Assert; import org.junit.Test; public abstract class UpdateIntegrationTest { + /** + * { + * "comment":"New Comment", + * "username":"test_user", + * "timestamp":1526401584951 + * } + */ + @Multiline + protected String commentOne; + + /** + * { + * "comment":"New Comment 2", + * "username":"test_user_2", + * "timestamp":1526401584952 + * } + */ + @Multiline + protected String commentTwo; + private static final int MAX_RETRIES = 10; private static final int SLEEP_MS = 500; protected static final String SENSOR_NAME = "test"; private static final String CF = "p"; - protected static MultiIndexDao dao; + private MultiIndexDao dao; @Test public void test() throws Exception { @@ -68,7 +99,7 @@ public abstract class UpdateIntegrationTest { put("new-field", "metron"); }}; String guid = "" + message0.get(Constants.GUID); - dao.replace(new ReplaceRequest(){{ + getDao().replace(new ReplaceRequest(){{ setReplacement(message0); setGuid(guid); setSensorType(SENSOR_NAME); @@ -76,8 +107,7 @@ public abstract class UpdateIntegrationTest { }}, Optional.empty()); Assert.assertEquals(1, getMockHTable().size()); - Document doc = dao.getLatest(guid, SENSOR_NAME); - Assert.assertEquals(message0, doc.getDocument()); + findUpdatedDoc(message0, guid, SENSOR_NAME); { //ensure hbase is up to date Get g = new Get(HBaseDao.Key.toBytes(new HBaseDao.Key(guid, SENSOR_NAME))); @@ -99,7 +129,7 @@ public abstract class UpdateIntegrationTest { .filter(d -> message0.get("new-field").equals(d.get("new-field"))) .count(); } - Assert.assertNotEquals("Elasticsearch is not updated!", cnt, 0); + Assert.assertNotEquals("Data store is not updated!", cnt, 0); } } //modify the same message and modify the new field @@ -108,15 +138,16 @@ public abstract class UpdateIntegrationTest { put("new-field", "metron2"); }}; String guid = "" + message0.get(Constants.GUID); - dao.replace(new ReplaceRequest(){{ + getDao().replace(new ReplaceRequest(){{ setReplacement(message0); setGuid(guid); setSensorType(SENSOR_NAME); setIndex(getIndexName()); }}, Optional.empty()); Assert.assertEquals(1, getMockHTable().size()); - Document doc = dao.getLatest(guid, SENSOR_NAME); + Document doc = getDao().getLatest(guid, SENSOR_NAME); Assert.assertEquals(message0, doc.getDocument()); + findUpdatedDoc(message0, guid, SENSOR_NAME); { //ensure hbase is up to date Get g = new Get(HBaseDao.Key.toBytes(new HBaseDao.Key(guid, SENSOR_NAME))); @@ -141,11 +172,133 @@ public abstract class UpdateIntegrationTest { .count(); } - Assert.assertNotEquals("Index is not updated!", cnt, 0); + Assert.assertNotEquals("Data store is not updated!", cnt, 0); } } } + @Test + public void testAddCommentAndPatch() throws Exception { + Map<String, Object> fields = new HashMap<>(); + fields.put("guid", "add_comment"); + fields.put("source.type", SENSOR_NAME); + + Document document = new Document(fields, "add_comment", SENSOR_NAME, 1526306463050L); + getDao().update(document, Optional.of(SENSOR_NAME)); + findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME); + + addAlertComment("add_comment", "New Comment", "test_user", 1526306463050L); + // Ensure we have the first comment + ArrayList<AlertComment> comments = new ArrayList<>(); + comments.add(new AlertComment("New Comment", "test_user", 1526306463050L)); + document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect( + Collectors.toList())); + findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME); + + List<Map<String, Object>> patchList = new ArrayList<>(); + Map<String, Object> patch = new HashMap<>(); + patch.put("op", "add"); + patch.put("path", "/project"); + patch.put("value", "metron"); + patchList.add(patch); + + PatchRequest pr = new PatchRequest(); + pr.setGuid("add_comment"); + pr.setIndex(SENSOR_NAME); + pr.setSensorType(SENSOR_NAME); + pr.setPatch(patchList); + getDao().patch(getDao(), pr, Optional.of(new Date().getTime())); + + document.getDocument().put("project", "metron"); + findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME); + } + + @Test + @SuppressWarnings("unchecked") + public void testRemoveComments() throws Exception { + Map<String, Object> fields = new HashMap<>(); + fields.put("guid", "add_comment"); + fields.put("source.type", SENSOR_NAME); + + Document document = new Document(fields, "add_comment", SENSOR_NAME, 1526401584951L); + getDao().update(document, Optional.of(SENSOR_NAME)); + findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME); + + addAlertComment("add_comment", "New Comment", "test_user", 1526401584951L); + // Ensure we have the first comment + ArrayList<AlertComment> comments = new ArrayList<>(); + comments.add(new AlertComment("New Comment", "test_user", 1526401584951L)); + document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect( + Collectors.toList())); + findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME); + + addAlertComment("add_comment", "New Comment 2", "test_user_2", 1526401584952L); + // Ensure we have the second comment + comments.add(new AlertComment("New Comment 2", "test_user_2", 1526401584952L)); + document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect( + Collectors.toList())); + findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME); + + removeAlertComment("add_comment", "New Comment 2", "test_user_2", 1526401584952L); + // Ensure we only have the first comments + comments = new ArrayList<>(); + comments.add(new AlertComment(commentOne)); + document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect( + Collectors.toList())); + findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME); + + removeAlertComment("add_comment", "New Comment", "test_user", 1526401584951L); + // Ensure we have no comments + document.getDocument().remove(COMMENTS_FIELD); + findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME); + } + + protected void addAlertComment(String guid, String comment, String username, long timestamp) + throws IOException { + CommentAddRemoveRequest request = buildAlertRequest(guid, comment, username, timestamp); + getDao().addCommentToAlert(request); + } + + protected void removeAlertComment(String guid, String comment, String username, long timestamp) + throws IOException { + CommentAddRemoveRequest request = buildAlertRequest(guid, comment, username, timestamp); + getDao().removeCommentFromAlert(request); + } + + private CommentAddRemoveRequest buildAlertRequest(String guid, String comment, String username, + long timestamp) { + CommentAddRemoveRequest request = new CommentAddRemoveRequest(); + request.setGuid(guid); + request.setComment(comment); + request.setUsername(username); + request.setTimestamp(timestamp); + request.setSensorType(SENSOR_NAME); + return request; + } + + protected void findUpdatedDoc(Map<String, Object> message0, String guid, String sensorType) + throws InterruptedException, IOException, OriginalNotFoundException { + for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) { + Document doc = getDao().getLatest(guid, sensorType); + if (doc != null && message0.equals(doc.getDocument())) { + return; + } + if (t == MAX_RETRIES -1) { + MapUtils.debugPrint(System.out, "Expected", message0); + MapUtils.debugPrint(System.out, "actual", doc.getDocument()); + } + } + throw new OriginalNotFoundException("Count not find " + guid + " after " + MAX_RETRIES + " tries"); + } + + protected IndexDao getDao() { + return dao; + } + + protected void setDao(MultiIndexDao dao) { + this.dao = dao; + } + protected abstract String getIndexName(); protected abstract MockHTable getMockHTable(); protected abstract void addTestData(String indexName, String sensorType, List<Map<String,Object>> docs) throws Exception; http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java index 2d620d9..7028b75 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java @@ -58,6 +58,7 @@ import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus; import org.apache.metron.indexing.dao.metaalert.MetaScores; import org.apache.metron.indexing.dao.search.GetRequest; import org.apache.metron.indexing.dao.search.InvalidSearchException; +import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.update.PatchRequest; import org.json.simple.JSONArray; @@ -146,6 +147,22 @@ public class AbstractLuceneMetaAlertUpdateDaoTest { } @Override + public void addCommentToAlert(CommentAddRemoveRequest request) { + } + + @Override + public void removeCommentFromAlert(CommentAddRemoveRequest request) { + } + + @Override + public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) { + } + + @Override + public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) { + } + + @Override public void patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request, Optional<Long> timestamp) { } http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java index aa32aa0..da74d46 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java @@ -20,6 +20,7 @@ package org.apache.metron.indexing.integration; import static org.apache.metron.indexing.dao.HBaseDao.HBASE_CF; import static org.apache.metron.indexing.dao.HBaseDao.HBASE_TABLE; +import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD; import java.io.IOException; import java.util.ArrayList; @@ -29,12 +30,14 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; - -import org.apache.commons.codec.binary.Hex; import org.apache.metron.hbase.mock.MockHBaseTableProvider; +import org.apache.metron.hbase.mock.MockHTable; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.HBaseDao; import org.apache.metron.indexing.dao.IndexDao; +import org.apache.metron.indexing.dao.MultiIndexDao; +import org.apache.metron.indexing.dao.UpdateIntegrationTest; +import org.apache.metron.indexing.dao.search.AlertComment; import org.apache.metron.indexing.dao.search.GetRequest; import org.apache.metron.indexing.dao.update.Document; import org.junit.After; @@ -42,7 +45,7 @@ import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; -public class HBaseDaoIntegrationTest { +public class HBaseDaoIntegrationTest extends UpdateIntegrationTest { private static final String TABLE_NAME = "metron_update"; private static final String COLUMN_FAMILY = "cf"; @@ -150,6 +153,13 @@ public class HBaseDaoIntegrationTest { Assert.assertFalse("Result size should be 12 but was greater", results.hasNext()); } + @Override + public void test() { + // The main test ensures a variety of things not implemented by HBase run alongside + // HBaseDao itself. + // Therefore, just don't do anything for this test. + } + protected List<Document> buildAlerts(int count) throws IOException { List<Document> alerts = new ArrayList<>(); for (int i = 0; i < count; ++i) { @@ -161,4 +171,67 @@ public class HBaseDaoIntegrationTest { return alerts; } + @Test + @SuppressWarnings("unchecked") + public void testRemoveComments() throws Exception { + Map<String, Object> fields = new HashMap<>(); + fields.put("guid", "add_comment"); + fields.put("source.type", SENSOR_NAME); + + Document document = new Document(fields, "add_comment", SENSOR_NAME, 1526401584951L); + hbaseDao.update(document, Optional.of(SENSOR_NAME)); + findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME); + + addAlertComment("add_comment", "New Comment", "test_user", 1526401584951L); + // Ensure we have the first comment + ArrayList<AlertComment> comments = new ArrayList<>(); + comments.add(new AlertComment("New Comment", "test_user", 1526401584951L)); + document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect( + Collectors.toList())); + findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME); + + addAlertComment("add_comment", "New Comment 2", "test_user_2", 1526401584952L); + // Ensure we have the second comment + comments.add(new AlertComment("New Comment 2", "test_user_2", 1526401584952L)); + document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect( + Collectors.toList())); + findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME); + + removeAlertComment("add_comment", "New Comment 2", "test_user_2", 1526401584952L); + // Ensure we only have the first comments + comments = new ArrayList<>(); + comments.add(new AlertComment(commentOne)); + document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect( + Collectors.toList())); + findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME); + + removeAlertComment("add_comment", "New Comment", "test_user", 1526401584951L); + // Ensure we have no comments + document.getDocument().remove(COMMENTS_FIELD); + findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME); + } + + @Override + protected IndexDao getDao() { + return hbaseDao; + } + + @Override + protected String getIndexName() { + return null; + } + + @Override + protected MockHTable getMockHTable() { + return null; + } + + @Override + protected void addTestData(String indexName, String sensorType, List<Map<String, Object>> docs) { + } + + @Override + protected List<Map<String, Object>> getIndexedTestData(String indexName, String sensorType) { + return null; + } } http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml b/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml index ca69304..1326dfc 100644 --- a/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml +++ b/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml @@ -677,6 +677,9 @@ <dynamicField name="*.reason" type="string" multiValued="false" docValues="true"/> <dynamicField name="*.name" type="string" multiValued="false" docValues="true"/> + <!-- Comments field required for the UI --> + <field name="comments" type="string" indexed="true" stored="true" multiValued="true"/> + <!-- Metaalerts Field --> <field name="metaalerts" type="string" multiValued="true" indexed="true" stored="true"/> http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml b/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml index 82d0320..84855df 100644 --- a/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml +++ b/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml @@ -70,6 +70,9 @@ <dynamicField name="*.reason" type="string" multiValued="false" docValues="true"/> <dynamicField name="*.name" type="string" multiValued="false" docValues="true"/> + <!-- Comments field required for the UI --> + <field name="comments" type="string" indexed="true" stored="true" multiValued="true"/> + <!-- Metaalerts Field --> <field name="metaalerts" type="string" multiValued="true" indexed="true" stored="true"/> http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml b/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml index fc8e641..5555a14 100644 --- a/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml +++ b/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml @@ -76,6 +76,9 @@ <dynamicField name="*.reason" type="string" multiValued="false" docValues="true"/> <dynamicField name="*.name" type="string" multiValued="false" docValues="true"/> + <!-- Comments field required for the UI --> + <field name="comments" type="string" indexed="true" stored="true" multiValued="true"/> + <!-- Metaalerts Field --> <field name="metaalerts" type="string" multiValued="true" indexed="true" stored="true"/>