http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java index d461792..7db0ab5 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java @@ -22,13 +22,16 @@ import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER; import com.google.common.base.Splitter; import java.io.IOException; import java.lang.invoke.MethodHandles; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Function; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.ColumnMetadataDao; import org.apache.metron.indexing.dao.IndexDao; import org.apache.metron.indexing.dao.RetrieveLatestDao; +import org.apache.metron.indexing.dao.search.AlertComment; import org.apache.metron.indexing.dao.search.FieldType; import org.apache.metron.indexing.dao.search.GetRequest; import org.apache.metron.indexing.dao.search.GroupRequest; @@ -36,6 +39,7 @@ import org.apache.metron.indexing.dao.search.GroupResponse; import org.apache.metron.indexing.dao.search.InvalidSearchException; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; +import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.update.OriginalNotFoundException; import org.apache.metron.indexing.dao.update.PatchRequest; @@ -88,12 +92,21 @@ public class SolrDao implements IndexDao { this.accessConfig = config; this.client = getSolrClient(getZkHosts()); this.solrSearchDao = new SolrSearchDao(this.client, this.accessConfig); - this.solrUpdateDao = new SolrUpdateDao(this.client, this.accessConfig); this.solrRetrieveLatestDao = new SolrRetrieveLatestDao(this.client); + this.solrUpdateDao = new SolrUpdateDao(this.client, this.solrRetrieveLatestDao, this.accessConfig); this.solrColumnMetadataDao = new SolrColumnMetadataDao(this.client); } } + public Optional<String> getIndex(String sensorName, Optional<String> index) { + if (index.isPresent()) { + return index; + } else { + String realIndex = accessConfig.getIndexSupplier().apply(sensorName); + return Optional.ofNullable(realIndex); + } + } + @Override public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException { return this.solrSearchDao.search(searchRequest); @@ -125,6 +138,16 @@ public class SolrDao implements IndexDao { } @Override + public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException { + this.solrUpdateDao.addCommentToAlert(request); + } + + @Override + public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException { + this.solrUpdateDao.removeCommentFromAlert(request); + } + + @Override public void patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request, Optional<Long> timestamp) throws OriginalNotFoundException, IOException { @@ -136,6 +159,18 @@ public class SolrDao implements IndexDao { return this.solrColumnMetadataDao.getColumnMetadata(indices); } + @Override + public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) + throws IOException { + this.solrUpdateDao.addCommentToAlert(request, latest); + } + + @Override + public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) + throws IOException { + this.solrUpdateDao.removeCommentFromAlert(request, latest); + } + /** * Builds a Solr client using the ZK hosts from the global config. * @return SolrClient @@ -170,7 +205,7 @@ public class SolrDao implements IndexDao { return solrSearchDao; } - public SolrSearchDao getSolrUpdateDao() { - return solrSearchDao; + public SolrUpdateDao getSolrUpdateDao() { + return solrUpdateDao; } }
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java index e65700f..8b37a49 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java @@ -41,6 +41,7 @@ import org.apache.metron.indexing.dao.search.InvalidCreateException; import org.apache.metron.indexing.dao.search.InvalidSearchException; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; +import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.update.OriginalNotFoundException; import org.apache.metron.indexing.dao.update.PatchRequest; @@ -222,4 +223,24 @@ public class SolrMetaAlertDao implements MetaAlertDao { throws IOException { return metaAlertUpdateDao.updateMetaAlertStatus(metaAlertGuid, status); } + + @Override + public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException { + solrDao.addCommentToAlert(request); + } + + @Override + public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException { + solrDao.removeCommentFromAlert(request); + } + + @Override + public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException { + solrDao.addCommentToAlert(request, latest); + } + + @Override + public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException { + solrDao.removeCommentFromAlert(request, latest); + } } http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java index 2a66d47..b96bbc6 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java @@ -42,6 +42,7 @@ import org.apache.metron.indexing.dao.search.InvalidCreateException; import org.apache.metron.indexing.dao.search.InvalidSearchException; import org.apache.metron.indexing.dao.search.SearchResponse; import org.apache.metron.indexing.dao.search.SearchResult; +import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.update.UpdateDao; import org.apache.solr.client.solrj.SolrClient; @@ -182,6 +183,28 @@ public class SolrMetaAlertUpdateDao extends AbstractLuceneMetaAlertUpdateDao imp } } + @Override + public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException { + getUpdateDao().addCommentToAlert(request); + } + + @Override + public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException { + getUpdateDao().removeCommentFromAlert(request); + } + + @Override + public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) + throws IOException { + getUpdateDao().addCommentToAlert(request, latest); + } + + @Override + public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) + throws IOException { + getUpdateDao().removeCommentFromAlert(request, latest); + } + protected boolean replaceAlertInMetaAlert(Document metaAlert, Document alert) { boolean metaAlertUpdated = removeAlertsFromMetaAlert(metaAlert, Collections.singleton(alert.getGuid())); http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java index 8578bfb..b3bc564 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java @@ -45,6 +45,7 @@ public class SolrRetrieveLatestDao implements RetrieveLatestDao { @Override public Document getLatest(String guid, String collection) throws IOException { + try { SolrDocument solrDocument = client.getById(collection, guid); if (solrDocument == null) { http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java index 4a8d482..d978ec9 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java @@ -17,10 +17,24 @@ */ package org.apache.metron.solr.dao; +import static org.apache.metron.common.Constants.SENSOR_TYPE; +import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD; + import com.fasterxml.jackson.core.JsonProcessingException; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.metron.common.Constants; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -29,6 +43,8 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.indexing.dao.AccessConfig; +import org.apache.metron.indexing.dao.search.AlertComment; +import org.apache.metron.indexing.dao.search.GetRequest; import org.apache.metron.indexing.dao.search.Group; import org.apache.metron.indexing.dao.search.GroupOrder; import org.apache.metron.indexing.dao.search.GroupOrderType; @@ -53,6 +69,7 @@ import org.apache.solr.client.solrj.response.PivotField; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; +import org.json.simple.parser.ParseException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java index 7c56169..7f5a4ed 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java @@ -17,19 +17,24 @@ */ package org.apache.metron.solr.dao; +import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD; + import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.function.Function; - +import java.util.stream.Collectors; import org.apache.metron.indexing.dao.AccessConfig; +import org.apache.metron.indexing.dao.search.AlertComment; +import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.update.UpdateDao; import org.apache.solr.client.solrj.SolrClient; @@ -39,23 +44,24 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class SolrUpdateDao implements UpdateDao { - private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private transient SolrClient client; private AccessConfig config; private Function<String, String> indexSupplier; - public SolrUpdateDao(SolrClient client, AccessConfig config) { + private transient SolrRetrieveLatestDao retrieveLatestDao; + + public SolrUpdateDao(SolrClient client, SolrRetrieveLatestDao retrieveLatestDao, AccessConfig config) { this.client = client; + this.retrieveLatestDao = retrieveLatestDao; this.config = config; this.indexSupplier = config.getIndexSupplier(); } private Optional<String> getIndex(String sensorName, Optional<String> index) { - if(index.isPresent()) { - return Optional.ofNullable(index.get()); - } - else { + if (index.isPresent()) { + return index; + } else { String realIndex = indexSupplier.apply(sensorName); return Optional.ofNullable(realIndex); } @@ -63,8 +69,17 @@ public class SolrUpdateDao implements UpdateDao { @Override public void update(Document update, Optional<String> rawIndex) throws IOException { + Document newVersion = update; + // Handle any case where we're given comments in Map form, instead of raw String + Object commentsObj = update.getDocument().get(COMMENTS_FIELD); + if ( commentsObj instanceof List && + ((List<Object>) commentsObj).size() > 0 && + ((List<Object>) commentsObj).get(0) instanceof Map) { + newVersion = new Document(update); + convertCommentsToRaw(newVersion.getDocument()); + } try { - SolrInputDocument solrInputDocument = SolrUtilities.toSolrInputDocument(update); + SolrInputDocument solrInputDocument = SolrUtilities.toSolrInputDocument(newVersion); Optional<String> index = getIndex(update.getSensorType(), rawIndex); if (index.isPresent()) { this.client.add(index.get(), solrInputDocument); @@ -113,4 +128,87 @@ public class SolrUpdateDao implements UpdateDao { throw new IOException(e); } } + + @Override + public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException { + Document latest = retrieveLatestDao.getLatest(request.getGuid(), request.getSensorType()); + addCommentToAlert(request, latest); + } + + @Override + public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException { + if (latest == null) { + return; + } + + @SuppressWarnings("unchecked") + List<Map<String, Object>> comments = (List<Map<String, Object>>) latest.getDocument() + .getOrDefault(COMMENTS_FIELD, new ArrayList<>()); + List<Map<String, Object>> originalComments = new ArrayList<>(comments); + + // Convert all comments back to raw JSON before updating. + List<String> commentStrs = new ArrayList<>(); + for (Map<String, Object> comment : originalComments) { + commentStrs.add(new AlertComment(comment).asJson()); + } + commentStrs.add(new AlertComment( + request.getComment(), + request.getUsername(), + request.getTimestamp() + ).asJson()); + + Document newVersion = new Document(latest); + newVersion.getDocument().put(COMMENTS_FIELD, commentStrs); + update(newVersion, Optional.empty()); + } + + @Override + public void removeCommentFromAlert(CommentAddRemoveRequest request) + throws IOException { + Document latest = retrieveLatestDao.getLatest(request.getGuid(), request.getSensorType()); + removeCommentFromAlert(request, latest); + } + + @Override + public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) + throws IOException { + if (latest == null) { + return; + } + + @SuppressWarnings("unchecked") + List<Map<String, Object>> commentMap = (List<Map<String, Object>>) latest.getDocument() + .get(COMMENTS_FIELD); + // Can't remove anything if there's nothing there + if (commentMap == null) { + LOG.debug("Provided alert had no comments to be able to remove from"); + return; + } + List<Map<String, Object>> originalComments = new ArrayList<>(commentMap); + List<AlertComment> comments = new ArrayList<>(); + for (Map<String, Object> commentStr : originalComments) { + comments.add(new AlertComment(commentStr)); + } + + comments.remove( + new AlertComment(request.getComment(), request.getUsername(), request.getTimestamp())); + List<String> commentsAsJson = comments.stream().map(AlertComment::asJson) + .collect(Collectors.toList()); + Document newVersion = new Document(latest); + newVersion.getDocument().put(COMMENTS_FIELD, commentsAsJson); + update(newVersion, Optional.empty()); + } + + public void convertCommentsToRaw(Map<String,Object> source) { + @SuppressWarnings("unchecked") + List<Map<String, Object>> comments = (List<Map<String, Object>>) source.get(COMMENTS_FIELD); + if (comments == null || comments.isEmpty()) { + return; + } + List<String> asJson = new ArrayList<>(); + for (Map<String, Object> comment : comments) { + asJson.add((new AlertComment(comment)).asJson()); + } + source.put(COMMENTS_FIELD, asJson); + } } http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java index 616bd1a..88146b0 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java @@ -18,18 +18,23 @@ package org.apache.metron.solr.dao; +import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Function; +import java.util.stream.Collectors; import org.apache.metron.common.Constants; import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants; +import org.apache.metron.indexing.dao.search.AlertComment; import org.apache.metron.indexing.dao.search.SearchResult; import org.apache.metron.indexing.dao.update.Document; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrInputDocument; +import org.json.simple.parser.ParseException; public class SolrUtilities { @@ -53,6 +58,34 @@ public class SolrUtilities { solrDocument.getFieldNames().stream() .filter(name -> !name.equals(SolrDao.VERSION_FIELD)) .forEach(name -> document.put(name, solrDocument.getFieldValue(name))); + + reformatComments(solrDocument, document); + insertChildAlerts(solrDocument, document); + + return new Document(document, + (String) solrDocument.getFieldValue(Constants.GUID), + (String) solrDocument.getFieldValue(Constants.SENSOR_TYPE), 0L); + } + + protected static void reformatComments(SolrDocument solrDocument, Map<String, Object> document) { + // Make sure comments are in the proper format + @SuppressWarnings("unchecked") + List<String> commentStrs = (List<String>) solrDocument.get(COMMENTS_FIELD); + if (commentStrs != null) { + try { + List<AlertComment> comments = new ArrayList<>(); + for (String commentStr : commentStrs) { + comments.add(new AlertComment(commentStr)); + } + document.put(COMMENTS_FIELD, + comments.stream().map(AlertComment::asMap).collect(Collectors.toList())); + } catch (ParseException e) { + throw new IllegalStateException("Unable to parse comment", e); + } + } + } + + protected static void insertChildAlerts(SolrDocument solrDocument, Map<String, Object> document) { // Make sure to put child alerts in if (solrDocument.hasChildDocuments() && solrDocument .getFieldValue(Constants.SENSOR_TYPE) @@ -68,9 +101,6 @@ public class SolrUtilities { document.put(MetaAlertConstants.ALERT_FIELD, childDocuments); } - return new Document(document, - (String) solrDocument.getFieldValue(Constants.GUID), - (String) solrDocument.getFieldValue(Constants.SENSOR_TYPE), 0L); } public static SolrInputDocument toSolrInputDocument(Document document) { http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java index 076fb20..85523b2 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java @@ -35,6 +35,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; import org.apache.metron.indexing.dao.AccessConfig; +import org.apache.metron.indexing.dao.IndexDao; +import org.apache.metron.indexing.dao.search.AlertComment; import org.apache.metron.indexing.dao.search.GetRequest; import org.apache.metron.indexing.dao.search.GroupRequest; import org.apache.metron.indexing.dao.search.SearchRequest; @@ -107,9 +109,10 @@ public class SolrDaoTest { solrDao = spy(new SolrDao()); doReturn(client).when(solrDao).getSolrClient(Collections.singletonList("zookeeper:2181")); whenNew(SolrSearchDao.class).withArguments(client, accessConfig).thenReturn(solrSearchDao); - whenNew(SolrUpdateDao.class).withArguments(client, accessConfig).thenReturn(solrUpdateDao); whenNew(SolrRetrieveLatestDao.class).withArguments(client) .thenReturn(solrRetrieveLatestDao); + whenNew(SolrUpdateDao.class).withArguments(client, solrRetrieveLatestDao, accessConfig) + .thenReturn(solrUpdateDao); whenNew(SolrColumnMetadataDao.class).withArguments(client) .thenReturn(solrColumnMetadataDao); http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java index 4571b15..43bf1b1 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java @@ -39,8 +39,8 @@ import org.apache.metron.indexing.dao.search.GroupResponse; import org.apache.metron.indexing.dao.search.InvalidCreateException; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; +import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.Document; -import org.apache.metron.indexing.dao.update.OriginalNotFoundException; import org.apache.metron.indexing.dao.update.PatchRequest; import org.junit.BeforeClass; import org.junit.Test; @@ -94,6 +94,22 @@ public class SolrMetaAlertDaoTest { } @Override + public void addCommentToAlert(CommentAddRemoveRequest request) { + } + + @Override + public void removeCommentFromAlert(CommentAddRemoveRequest request) { + } + + @Override + public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) { + } + + @Override + public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) { + } + + @Override public void patch(RetrieveLatestDao dao, PatchRequest request, Optional<Long> timestamp) { } http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java index 41fe7f9..7de02c1 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java @@ -17,11 +17,32 @@ */ package org.apache.metron.solr.dao; -import org.apache.metron.common.configuration.Configurations; +import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD; +import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER; +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.argThat; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.powermock.api.mockito.PowerMockito.doReturn; +import static org.powermock.api.mockito.PowerMockito.spy; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.metron.common.Constants; import org.apache.metron.common.configuration.IndexingConfigurations; import org.apache.metron.common.zookeeper.ConfigurationsCache; import org.apache.metron.indexing.dao.AccessConfig; +import org.apache.metron.indexing.dao.search.AlertComment; import org.apache.metron.indexing.dao.update.Document; +import org.apache.metron.indexing.dao.update.OriginalNotFoundException; +import org.apache.metron.indexing.dao.update.PatchRequest; import org.apache.metron.indexing.util.IndexingCacheUtil; import org.apache.metron.solr.matcher.SolrInputDocumentListMatcher; import org.apache.metron.solr.matcher.SolrInputDocumentMatcher; @@ -34,22 +55,9 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; -import org.mockito.Matchers; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER; -import static org.mockito.Matchers.argThat; -import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - @RunWith(PowerMockRunner.class) @PrepareForTest({CollectionAdminRequest.class}) public class SolrUpdateDaoTest { @@ -58,6 +66,7 @@ public class SolrUpdateDaoTest { public final ExpectedException exception = ExpectedException.none(); private SolrClient client; + private SolrRetrieveLatestDao solrRetrieveLatestDao; private SolrUpdateDao solrUpdateDao; private static AccessConfig accessConfig = new AccessConfig(); @@ -86,7 +95,8 @@ public class SolrUpdateDaoTest { @Before public void setUp() throws Exception { client = mock(SolrClient.class); - solrUpdateDao = new SolrUpdateDao(client, accessConfig); + solrRetrieveLatestDao = new SolrRetrieveLatestDao(client); + solrUpdateDao = new SolrUpdateDao(client, solrRetrieveLatestDao, accessConfig); } @Test @@ -174,4 +184,58 @@ public class SolrUpdateDaoTest { verify(client).add(eq("snort"), argThat(new SolrInputDocumentListMatcher(Arrays.asList(snortSolrInputDocument1, snortSolrInputDocument2)))); } + @Test + public void testConvertCommentsToRaw() { + List<Map<String, Object>> commentList = new ArrayList<>(); + Map<String, Object> comments = new HashMap<>(); + comments.put("comment", "test comment"); + comments.put("username", "test username"); + comments.put("timestamp", 1526424323279L); + commentList.add(comments); + + Map<String, Object> document = new HashMap<>(); + document.put("testField", "testValue"); + document.put(COMMENTS_FIELD, commentList); + solrUpdateDao.convertCommentsToRaw(document); + + @SuppressWarnings("unchecked") + List<String> actualComments = (List<String>) document.get(COMMENTS_FIELD); + String expectedComment = "{\"comment\":\"test comment\",\"username\":\"test username\",\"timestamp\":1526424323279}"; + assertEquals(expectedComment, actualComments.get(0)); + assertEquals(1, actualComments.size()); + assertEquals("testValue", document.get("testField")); + } + + @Test + public void getPatchedDocument() throws IOException, OriginalNotFoundException { + // Create the document to be patched. Including comments + Map<String, Object> latestDoc = new HashMap<>(); + latestDoc.put(Constants.GUID, "guid"); + List<Map<String, Object>> comments = new ArrayList<>(); + comments.add(new AlertComment("comment", "user", 0L).asMap()); + comments.add(new AlertComment("comment_2", "user_2", 0L).asMap()); + latestDoc.put(COMMENTS_FIELD, comments); + Document latest = new Document(latestDoc, "guid", "bro", 0L); + + SolrRetrieveLatestDao retrieveLatestDao = spy(new SolrRetrieveLatestDao(null)); + doReturn(latest).when(retrieveLatestDao).getLatest("guid", "bro"); + + // Create the patch + PatchRequest request = new PatchRequest(); + request.setIndex("bro"); + request.setSensorType("bro"); + request.setGuid("guid"); + List<Map<String, Object>> patchList = new ArrayList<>(); + Map<String, Object> patch = new HashMap<>(); + patch.put("op", "add"); + patch.put("path", "/project"); + patch.put("value", "metron"); + patchList.add(patch); + request.setPatch(patchList); + Document actual = solrUpdateDao.getPatchedDocument(retrieveLatestDao, request, Optional.of(0L)); + + // Add the patch to our original document + latest.getDocument().put("project", "metron"); + assertEquals(actual, latest); + } } http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java index f09d7a8..4390fd1 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java @@ -110,8 +110,8 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest { // getColumnMetadata with only bro { Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("bro")); - // Don't test all 256, just test a sample of different fields - Assert.assertEquals(262, fieldTypes.size()); + // Don't test all fields, just test a sample of different fields + Assert.assertEquals(263, fieldTypes.size()); // Fields present in both with same type Assert.assertEquals(FieldType.TEXT, fieldTypes.get("guid")); @@ -147,7 +147,7 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest { // getColumnMetadata with only snort { Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("snort")); - Assert.assertEquals(32, fieldTypes.size()); + Assert.assertEquals(33, fieldTypes.size()); // Fields present in both with same type Assert.assertEquals(FieldType.TEXT, fieldTypes.get("guid")); http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java index 7faeeb9..c0697b8 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java @@ -26,8 +26,11 @@ import java.util.List; import java.util.Map; import java.util.Optional; import org.apache.commons.lang3.StringUtils; +import org.apache.curator.framework.CuratorFramework; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.metron.common.configuration.ConfigurationsUtils; +import org.apache.metron.common.zookeeper.ZKConfigurationsCache; import org.apache.metron.hbase.mock.MockHBaseTableProvider; import org.apache.metron.hbase.mock.MockHTable; import org.apache.metron.indexing.dao.AccessConfig; @@ -36,6 +39,7 @@ import org.apache.metron.indexing.dao.IndexDao; import org.apache.metron.indexing.dao.MultiIndexDao; import org.apache.metron.indexing.dao.UpdateIntegrationTest; import org.apache.metron.indexing.dao.update.Document; +import org.apache.metron.indexing.util.IndexingCacheUtil; import org.apache.metron.solr.dao.SolrDao; import org.apache.metron.solr.integration.components.SolrComponent; import org.junit.After; @@ -81,8 +85,16 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest { globalConfig.put(HBaseDao.HBASE_CF, CF); accessConfig.setGlobalConfigSupplier(() -> globalConfig); - dao = new MultiIndexDao(hbaseDao, new SolrDao()); + CuratorFramework client = ConfigurationsUtils + .getClient(solrComponent.getZookeeperUrl()); + client.start(); + ZKConfigurationsCache cache = new ZKConfigurationsCache(client); + cache.start(); + accessConfig.setIndexSupplier(IndexingCacheUtil.getIndexLookupFunction(cache, "solr")); + + MultiIndexDao dao = new MultiIndexDao(hbaseDao, new SolrDao()); dao.init(accessConfig); + setDao(dao); } @After @@ -136,9 +148,9 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest { fields.put("field.location_point", "48.5839,7.7455"); Document document = new Document(fields, "bro_1", SENSOR_NAME, 0L); - dao.update(document, Optional.of(SENSOR_NAME)); + getDao().update(document, Optional.of(SENSOR_NAME)); - Document indexedDocument = dao.getLatest("bro_1", SENSOR_NAME); + Document indexedDocument = getDao().getLatest("bro_1", SENSOR_NAME); // assert no extra expanded fields are included assertEquals(8, indexedDocument.getDocument().size()); @@ -155,10 +167,10 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest { documentMap.put("raw_message", hugeString); documentMap.put("raw_message_1", hugeStringTwo); Document errorDoc = new Document(documentMap, "error", "error", 0L); - dao.update(errorDoc, Optional.of("error")); + getDao().update(errorDoc, Optional.of("error")); // Ensure that the huge string is returned when not a string field - Document latest = dao.getLatest("error_guid", "error"); + Document latest = getDao().getLatest("error_guid", "error"); @SuppressWarnings("unchecked") String actual = (String) latest.getDocument().get("raw_message"); assertEquals(actual, hugeString); @@ -171,6 +183,6 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest { exception.expect(IOException.class); exception.expectMessage("Document contains at least one immense term in field=\"error_hash\""); - dao.update(errorDoc, Optional.of("error")); + getDao().update(errorDoc, Optional.of("error")); } } http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema b/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema index bb2de59..8340a36 100644 --- a/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema +++ b/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema @@ -44,6 +44,9 @@ <field name="score" type="pdouble" indexed="true" stored="true"/> + <!-- Comments field required for the UI --> + <field name="comments" type="string" indexed="true" stored="true" multiValued="true"/> + <dynamicField name="*" type="ignored" multiValued="false" docValues="true"/>