http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java new file mode 100644 index 0000000..2d620d9 --- /dev/null +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java @@ -0,0 +1,854 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.indexing.dao.metaalert.lucene; + +import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.ALERT_FIELD; +import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.GROUPS_FIELD; +import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.METAALERT_FIELD; +import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.METAALERT_TYPE; +import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.STATUS_FIELD; +import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.THREAT_FIELD_DEFAULT; +import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.THREAT_SORT_DEFAULT; +import static org.apache.metron.indexing.dao.metaalert.MetaAlertStatus.ACTIVE; +import static org.apache.metron.indexing.dao.metaalert.MetaAlertStatus.INACTIVE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.UUID; +import org.adrianwalker.multilinestring.Multiline; +import org.apache.commons.math.util.MathUtils; +import org.apache.metron.common.Constants; +import org.apache.metron.common.Constants.Fields; +import org.apache.metron.indexing.dao.IndexDao; +import org.apache.metron.indexing.dao.RetrieveLatestDao; +import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig; +import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants; +import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest; +import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse; +import org.apache.metron.indexing.dao.metaalert.MetaAlertRetrieveLatestDao; +import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus; +import org.apache.metron.indexing.dao.metaalert.MetaScores; +import org.apache.metron.indexing.dao.search.GetRequest; +import org.apache.metron.indexing.dao.search.InvalidSearchException; +import org.apache.metron.indexing.dao.update.Document; +import org.apache.metron.indexing.dao.update.PatchRequest; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class AbstractLuceneMetaAlertUpdateDaoTest { + + @Mock + IndexDao indexDao; + + @Before + public void setup() { + dao = new TestLuceneMetaAlertUpdateDao(); + } + + private static final double EPS = 0.00001; + private static final String METAALERT_INDEX = "metaalert_index"; + private static final String METAALERT_GUID = "meta_0"; + private static final String DEFAULT_PREFIX = "child_"; + private static final MetaAlertConfig TEST_CONFIG = new MetaAlertConfig( + METAALERT_INDEX, + THREAT_FIELD_DEFAULT, + THREAT_SORT_DEFAULT, + Constants.SENSOR_TYPE + ); + + private static Map<String, Document> documents = new HashMap<>(); + + static { + Document active = new Document( + new HashMap<>(), + ACTIVE.getStatusString(), + METAALERT_TYPE, + 0L + ); + documents.put(ACTIVE.getStatusString(), active); + + Document inactive = new Document( + new HashMap<>(), + INACTIVE.getStatusString(), + METAALERT_TYPE, + 0L + ); + inactive.getDocument().put( + STATUS_FIELD, + INACTIVE.getStatusString() + ); + documents.put(INACTIVE.getStatusString(), inactive); + } + + TestMetaAlertRetrieveLatestDao retrieveLatestDao = new TestMetaAlertRetrieveLatestDao(); + + private class TestMetaAlertRetrieveLatestDao implements MetaAlertRetrieveLatestDao { + + @Override + public Document getLatest(String guid, String sensorType) { + return documents.get(guid); + } + + @Override + public Iterable<Document> getAllLatest(List<GetRequest> getRequests) { + return null; + } + } + + TestLuceneMetaAlertUpdateDao dao = new TestLuceneMetaAlertUpdateDao(); + + private class TestLuceneMetaAlertUpdateDao extends AbstractLuceneMetaAlertUpdateDao { + + TestLuceneMetaAlertUpdateDao() { + super(indexDao, retrieveLatestDao, TEST_CONFIG); + } + + @Override + public void update(Document update, Optional<String> index) { + } + + @Override + public void patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request, + Optional<Long> timestamp) { + } + + @Override + public MetaAlertCreateResponse createMetaAlert(MetaAlertCreateRequest request) { + return null; + } + + @Override + public boolean addAlertsToMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests) { + return false; + } + + @Override + public boolean updateMetaAlertStatus(String metaAlertGuid, MetaAlertStatus status) { + return false; + } + } + + /** + { + "guid": "meta_alert", + "index": "metaalert_index", + "patch": [ + { + "op": "add", + "path": "/alert", + "value": [] + } + ], + "sensorType": "metaalert" + } + */ + @Multiline + public static String alertPatchRequest; + + /** + { + "guid": "meta_alert", + "index": "metaalert_index", + "patch": [ + { + "op": "add", + "path": "/status", + "value": [] + } + ], + "sensorType": "metaalert" + } + */ + @Multiline + public static String statusPatchRequest; + + /** + { + "guid": "meta_alert", + "index": "metaalert_index", + "patch": [ + { + "op": "add", + "path": "/name", + "value": [] + } + ], + "sensorType": "metaalert" + } + */ + @Multiline + public static String namePatchRequest; + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Test(expected = UnsupportedOperationException.class) + public void testBatchUpdateThrowsException() { + dao.batchUpdate(null); + } + + @Test + @SuppressWarnings("unchecked") + public void testPatchNotAllowedAlert() throws ParseException { + PatchRequest pr = new PatchRequest(); + Map<String, Object> patch = (JSONObject) new JSONParser().parse(alertPatchRequest); + pr.setPatch(Collections.singletonList((JSONObject) ((JSONArray) patch.get("patch")).get(0))); + assertFalse(dao.isPatchAllowed(pr)); + } + + @Test + @SuppressWarnings("unchecked") + public void testPatchNotAllowedStatus() throws ParseException { + PatchRequest pr = new PatchRequest(); + Map<String, Object> patch = (JSONObject) new JSONParser().parse(statusPatchRequest); + pr.setPatch(Collections.singletonList((JSONObject) ((JSONArray) patch.get("patch")).get(0))); + assertFalse(dao.isPatchAllowed(pr)); + } + + @Test + @SuppressWarnings("unchecked") + public void testPatchAllowedName() throws ParseException { + PatchRequest pr = new PatchRequest(); + Map<String, Object> patch = (JSONObject) new JSONParser().parse(namePatchRequest); + pr.setPatch(Collections.singletonList((JSONObject) ((JSONArray) patch.get("patch")).get(0))); + assertTrue(dao.isPatchAllowed(pr)); + } + + @Test + public void testUpdateSingle() throws IOException { + Map<Document, Optional<String>> updates = new HashMap<>(); + Document document = new Document(new HashMap<>(), "guid", "sensor", 0L); + updates.put(document, Optional.empty()); + dao.update(updates); + verify(indexDao, times(1)).update(document, Optional.empty()); + } + + @Test + public void testUpdateMultiple() throws IOException { + Map<Document, Optional<String>> updates = new HashMap<>(); + Document documentOne = new Document(new HashMap<>(), "guid", "sensor", 0L); + updates.put(documentOne, Optional.empty()); + Document documentTwo = new Document(new HashMap<>(), "guid2", "sensor", 0L); + updates.put(documentTwo, Optional.empty()); + dao.update(updates); + verify(indexDao, times(1)).batchUpdate(updates); + } + + @Test + public void testBuildAddAlertToMetaAlertUpdatesEmpty() { + Document metaDoc = new Document( + new HashMap<>(), + METAALERT_GUID, + METAALERT_TYPE, + 0L + ); + metaDoc.getDocument().put( + ALERT_FIELD, + getRawMaps(buildChildAlerts(1, METAALERT_GUID, null)) + ); + Map<Document, Optional<String>> actual = dao + .buildAddAlertToMetaAlertUpdates(metaDoc, new ArrayList<>()); + assertEquals(0, actual.size()); + } + + @Test + public void testBuildAddAlertToMetaAlertUpdates() { + List<Document> alerts = buildChildAlerts(1, METAALERT_GUID, null); + + Document metaDoc = buildMetaAlert(alerts); + + List<Document> newAlerts = buildChildAlerts(2, null, "new_"); + Map<Document, Optional<String>> actual = dao + .buildAddAlertToMetaAlertUpdates(metaDoc, newAlerts); + assertEquals(3, actual.size()); + + HashMap<String, Object> expectedExistingAlert = new HashMap<>(); + expectedExistingAlert.put(Constants.GUID, "child_0"); + expectedExistingAlert.put(METAALERT_FIELD, Collections.singletonList(METAALERT_GUID)); + expectedExistingAlert.put(THREAT_FIELD_DEFAULT, 0.0f); + + List<Map<String, Object>> expectedAlerts = new ArrayList<>(); + expectedAlerts.add(expectedExistingAlert); + expectedAlerts.addAll(getRawMaps(newAlerts)); + + List<Double> scores = new ArrayList<>(); + scores.add(0.0d); + scores.add(0.0d); + scores.add(0.0d); + + Map<String, Object> expectedMetaAlertMap = new HashMap<>(); + expectedMetaAlertMap.put(Constants.GUID, METAALERT_GUID); + expectedMetaAlertMap.put(ALERT_FIELD, expectedAlerts); + expectedMetaAlertMap.put(THREAT_FIELD_DEFAULT, 0.0f); + + expectedMetaAlertMap.putAll(new MetaScores(scores).getMetaScores()); + Document expectedMetaAlertDoc = new Document(expectedMetaAlertMap, METAALERT_GUID, + METAALERT_TYPE, + 0L); + + Map<Document, Optional<String>> expected = new HashMap<>(); + expected.put(expectedMetaAlertDoc, Optional.of(METAALERT_INDEX)); + expected.put(newAlerts.get(0), Optional.empty()); + expected.put(newAlerts.get(1), Optional.empty()); + + assertTrue(updatesMapEquals(expected, actual)); + } + + @Test + public void testRemoveAlertsFromMetaAlert() throws IOException { + List<Document> alerts = buildChildAlerts(3, METAALERT_GUID, null); + + Document metaDoc = buildMetaAlert(alerts); + + List<Document> deletedAlerts = new ArrayList<>(); + deletedAlerts.add(alerts.get(0)); + deletedAlerts.add(alerts.get(2)); + + Map<Document, Optional<String>> actual = dao + .buildRemoveAlertsFromMetaAlert(metaDoc, deletedAlerts); + assertEquals(3, actual.size()); + + Map<String, Object> expectedDeletedAlert = new HashMap<>(); + expectedDeletedAlert.put(Constants.GUID, "child_0"); + expectedDeletedAlert.put(THREAT_FIELD_DEFAULT, 0.0f); + expectedDeletedAlert + .put(MetaAlertConstants.METAALERT_FIELD, new ArrayList<>()); + Document expectedDeletedDocument = new Document(expectedDeletedAlert, "child_0", "test", 0L); + + Map<String, Object> expectedDeletedAlert3 = new HashMap<>(); + expectedDeletedAlert3.put(Constants.GUID, "child_2"); + expectedDeletedAlert3.put(THREAT_FIELD_DEFAULT, 0.0f); + expectedDeletedAlert3 + .put(MetaAlertConstants.METAALERT_FIELD, new ArrayList<>()); + Document expectedDeletedDocument2 = new Document(expectedDeletedAlert3, "child_2", "test", 0L); + + List<Map<String, Object>> expectedAlerts = new ArrayList<>(); + expectedAlerts.add(alerts.get(1).getDocument()); + + Map<String, Object> expectedMetaAlertMap = new HashMap<>(); + expectedMetaAlertMap.put(Constants.GUID, METAALERT_GUID); + expectedMetaAlertMap.put(ALERT_FIELD, expectedAlerts); + expectedMetaAlertMap.put(THREAT_FIELD_DEFAULT, 0.0f); + expectedMetaAlertMap.putAll(new MetaScores(Collections.singletonList(0.0d)).getMetaScores()); + Document expectedMetaAlertDoc = new Document(expectedMetaAlertMap, METAALERT_GUID, + METAALERT_TYPE, + 0L); + + Map<Document, Optional<String>> expected = new HashMap<>(); + expected.put(expectedDeletedDocument, Optional.empty()); + expected.put(expectedDeletedDocument2, Optional.empty()); + expected.put(expectedMetaAlertDoc, Optional.of(METAALERT_INDEX)); + + assertTrue(updatesMapEquals(expected, actual)); + } + + @Test + public void testBuildRemoveAlertsFromMetaAlertThrowsException() throws Exception { + thrown.expect(IllegalStateException.class); + thrown.expectMessage("Removing these alerts will result in an empty meta alert. Empty meta alerts are not allowed."); + + List<Document> alerts = buildChildAlerts(1, METAALERT_GUID, null); + Document metaDoc = buildMetaAlert(alerts); + + dao.buildRemoveAlertsFromMetaAlert(metaDoc, alerts); + } + + @Test + public void testRemoveAlertsFromMetaAlertNoChildAlerts() { + Document empty = new Document(new HashMap<>(), "empty", METAALERT_TYPE, 0L); + boolean actual = dao.removeAlertsFromMetaAlert(empty, Collections.singletonList("child")); + assertFalse(actual); + } + + @Test + public void testRemoveAlertsFromMetaAlertEmptyRemoveList() { + Document metaDoc = new Document( + new HashMap<>(), + METAALERT_GUID, + METAALERT_TYPE, + 0L + ); + metaDoc.getDocument().put( + STATUS_FIELD, + ACTIVE.getStatusString() + ); + metaDoc.getDocument().put( + ALERT_FIELD, + new HashMap<String, Object>() {{ + put(Constants.GUID, "child_0"); + }} + ); + boolean actual = dao.removeAlertsFromMetaAlert(metaDoc, new ArrayList<>()); + assertFalse(actual); + } + + @Test + public void testRemoveAlertsFromMetaAlertEmptyRemoveSingle() { + Document metaDoc = new Document( + new HashMap<>(), + METAALERT_GUID, + METAALERT_TYPE, + 0L + ); + metaDoc.getDocument().put( + STATUS_FIELD, + ACTIVE.getStatusString() + ); + List<Map<String, Object>> alerts = new ArrayList<>(); + alerts.add(new HashMap<String, Object>() {{ + put(Constants.GUID, "child_0"); + }}); + metaDoc.getDocument().put( + ALERT_FIELD, + alerts + ); + boolean actual = dao.removeAlertsFromMetaAlert(metaDoc, Collections.singletonList("child_0")); + + Document expected = new Document( + new HashMap<>(), + METAALERT_GUID, + METAALERT_TYPE, + 0L + ); + expected.getDocument().put( + STATUS_FIELD, + ACTIVE.getStatusString() + ); + expected.getDocument().put(ALERT_FIELD, new ArrayList<>()); + assertTrue(actual); + assertEquals(expected, metaDoc); + } + + @Test + public void testBuildStatusChangeUpdatesToInactive() { + List<Document> alerts = buildChildAlerts(2, METAALERT_GUID, null); + + Map<String, Object> metaAlertMap = new HashMap<>(); + metaAlertMap.put(ALERT_FIELD, getRawMaps(alerts)); + metaAlertMap.put(Constants.GUID, METAALERT_GUID); + metaAlertMap.put(STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString()); + Document metaDoc = new Document( + metaAlertMap, + METAALERT_GUID, + METAALERT_TYPE, + 0L + ); + + Map<Document, Optional<String>> actual = dao + .buildStatusChangeUpdates(metaDoc, alerts, MetaAlertStatus.INACTIVE); + assertEquals(3, actual.size()); + + List<Document> expectedDeletedAlerts = buildChildAlerts(2, null, null); + List<Map<String, Object>> expectedAlerts = new ArrayList<>(); + expectedAlerts.add(alerts.get(0).getDocument()); + expectedAlerts.add(alerts.get(1).getDocument()); + + Map<String, Object> expectedMetaAlertMap = new HashMap<>(); + expectedMetaAlertMap.put(Constants.GUID, METAALERT_GUID); + expectedMetaAlertMap.put(ALERT_FIELD, expectedAlerts); + expectedMetaAlertMap.put(STATUS_FIELD, MetaAlertStatus.INACTIVE.getStatusString()); + Document expectedMetaAlertDoc = new Document(expectedMetaAlertMap, METAALERT_GUID, + METAALERT_TYPE, + 0L); + + Map<Document, Optional<String>> expected = new HashMap<>(); + expected.put(expectedMetaAlertDoc, Optional.of(METAALERT_INDEX)); + expected.put(expectedDeletedAlerts.get(0), Optional.empty()); + expected.put(expectedDeletedAlerts.get(1), Optional.empty()); + + assertTrue(updatesMapEquals(expected, actual)); + } + + @Test + public void testBuildStatusChangeUpdatesToActive() { + List<Document> alerts = buildChildAlerts(2, METAALERT_GUID, null); + + Map<String, Object> metaAlertMap = new HashMap<>(); + metaAlertMap.put(ALERT_FIELD, getRawMaps(alerts)); + metaAlertMap.put(Constants.GUID, METAALERT_GUID); + metaAlertMap.put(STATUS_FIELD, MetaAlertStatus.INACTIVE.getStatusString()); + Document metaDoc = new Document( + metaAlertMap, + METAALERT_GUID, + METAALERT_TYPE, + 0L + ); + + Map<Document, Optional<String>> actual = dao.buildStatusChangeUpdates( + metaDoc, + alerts, + MetaAlertStatus.ACTIVE + ); + + List<Map<String, Object>> expectedAlerts = new ArrayList<>(); + expectedAlerts.add(alerts.get(0).getDocument()); + expectedAlerts.add(alerts.get(1).getDocument()); + + Map<String, Object> expectedMetaAlertMap = new HashMap<>(); + expectedMetaAlertMap.put(ALERT_FIELD, expectedAlerts); + expectedMetaAlertMap.put(Constants.GUID, METAALERT_GUID); + expectedMetaAlertMap.put(STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString()); + Document expectedMetaAlertDoc = new Document( + expectedMetaAlertMap, + METAALERT_GUID, + METAALERT_TYPE, + 0L + ); + + Map<Document, Optional<String>> expected = new HashMap<>(); + expected.put(expectedMetaAlertDoc, Optional.of(METAALERT_INDEX)); + + assertTrue(updatesMapEquals(expected, actual)); + } + + @Test + public void testRemoveAlertsFromMetaAlertEmptyRemoveMultiple() { + Document metDoc = new Document(new HashMap<>(), METAALERT_GUID, METAALERT_TYPE, 0L); + metDoc.getDocument().put(STATUS_FIELD, ACTIVE.getStatusString()); + List<Document> alerts = buildChildAlerts(3, null, null); + metDoc.getDocument().put(ALERT_FIELD, getRawMaps(alerts)); + List<String> removeGuids = new ArrayList<>(); + removeGuids.add("child_0"); + removeGuids.add("child_2"); + removeGuids.add("child_doesn't_exist"); + + boolean actual = dao.removeAlertsFromMetaAlert(metDoc, removeGuids); + + // Build the expected metaalert + Document expected = new Document(new HashMap<>(), METAALERT_GUID, METAALERT_TYPE, 0L); + expected.getDocument().put(STATUS_FIELD, ACTIVE.getStatusString()); + List<Map<String, Object>> alertsExpected = new ArrayList<>(); + alertsExpected.add(new HashMap<String, Object>() {{ + put(METAALERT_FIELD, new ArrayList<>()); + put(Constants.GUID, "child_1"); + put(THREAT_FIELD_DEFAULT, 0.0f); + }} + ); + + expected.getDocument().put(ALERT_FIELD, alertsExpected); + assertEquals(expected, metDoc); + assertTrue(actual); + } + + @Test(expected = IllegalStateException.class) + public void testRemoveAlertsFromMetaAlertInactive() throws IOException { + dao.removeAlertsFromMetaAlert(INACTIVE.getStatusString(), null); + } + + @Test + public void testRemoveMetaAlertFromAlertSuccess() { + List<String> metaAlertGuids = new ArrayList<>(); + metaAlertGuids.add("metaalert1"); + metaAlertGuids.add("metaalert2"); + Map<String, Object> alertFields = new HashMap<>(); + alertFields.put(METAALERT_FIELD, metaAlertGuids); + Document alert = new Document(alertFields, "alert", "test", 0L); + + Document expected = new Document(new HashMap<>(), "alert", "test", 0L); + List<String> expectedMetaAlertGuids = new ArrayList<>(); + expectedMetaAlertGuids.add("metaalert2"); + expected.getDocument().put(METAALERT_FIELD, expectedMetaAlertGuids); + + boolean actual = dao.removeMetaAlertFromAlert("metaalert1", alert); + assertTrue(actual); + assertEquals(expected, alert); + } + + @Test + public void testRemoveMetaAlertFromAlertMissing() { + List<String> metaAlertGuids = new ArrayList<>(); + metaAlertGuids.add("metaalert1"); + metaAlertGuids.add("metaalert2"); + Map<String, Object> alertFields = new HashMap<>(); + alertFields.put(METAALERT_FIELD, metaAlertGuids); + Document alert = new Document(alertFields, "alert", "test", 0L); + + boolean actual = dao.removeMetaAlertFromAlert("metaalert3", alert); + assertFalse(actual); + } + + @Test + public void testAddMetaAlertToAlertEmpty() { + Map<String, Object> alertFields = new HashMap<>(); + alertFields.put(METAALERT_FIELD, new ArrayList<>()); + Document alert = new Document(alertFields, "alert", "test", 0L); + + Document expected = new Document(new HashMap<>(), "alert", "test", 0L); + List<String> expectedMetaAlertGuids = new ArrayList<>(); + expectedMetaAlertGuids.add("metaalert1"); + expected.getDocument().put(METAALERT_FIELD, expectedMetaAlertGuids); + + boolean actual = dao.addMetaAlertToAlert("metaalert1", alert); + assertTrue(actual); + assertEquals(expected, alert); + } + + @Test + public void testAddMetaAlertToAlertNonEmpty() { + List<String> metaAlertGuids = new ArrayList<>(); + metaAlertGuids.add("metaalert1"); + Map<String, Object> alertFields = new HashMap<>(); + alertFields.put(METAALERT_FIELD, metaAlertGuids); + Document alert = new Document(alertFields, "alert", "test", 0L); + + Document expected = new Document(new HashMap<>(), "alert", "test", 0L); + List<String> expectedMetaAlertGuids = new ArrayList<>(); + expectedMetaAlertGuids.add("metaalert1"); + expectedMetaAlertGuids.add("metaalert2"); + expected.getDocument().put(METAALERT_FIELD, expectedMetaAlertGuids); + + boolean actual = dao.addMetaAlertToAlert("metaalert2", alert); + assertTrue(actual); + assertEquals(expected, alert); + } + + @Test + public void testAddMetaAlertToAlertDuplicate() { + List<String> metaAlertGuids = new ArrayList<>(); + metaAlertGuids.add("metaalert1"); + Map<String, Object> alertFields = new HashMap<>(); + alertFields.put(METAALERT_FIELD, metaAlertGuids); + Document alert = new Document(alertFields, "alert", "test", 0L); + + boolean actual = dao.addMetaAlertToAlert("metaalert1", alert); + assertFalse(actual); + } + + @Test + public void testBuildCreateDocumentSingleAlert() { + List<String> groups = new ArrayList<>(); + groups.add("group_one"); + groups.add("group_two"); + + // Build the first response from the multiget + Map<String, Object> alertOne = new HashMap<>(); + alertOne.put(Constants.GUID, "alert_one"); + alertOne.put(THREAT_FIELD_DEFAULT, 10.0d); + List<Document> alerts = new ArrayList<Document>() {{ + add(new Document(alertOne, "", "", 0L)); + }}; + + // Actually build the doc + Document actual = dao.buildCreateDocument(alerts, groups, ALERT_FIELD); + + ArrayList<Map<String, Object>> alertList = new ArrayList<>(); + alertList.add(alertOne); + + Map<String, Object> actualDocument = actual.getDocument(); + assertEquals( + MetaAlertStatus.ACTIVE.getStatusString(), + actualDocument.get(STATUS_FIELD) + ); + assertEquals( + alertList, + actualDocument.get(ALERT_FIELD) + ); + assertEquals( + groups, + actualDocument.get(GROUPS_FIELD) + ); + + // Don't care about the result, just that it's a UUID. Exception will be thrown if not. + UUID.fromString((String) actualDocument.get(Constants.GUID)); + } + + @Test + public void testBuildCreateDocumentMultipleAlerts() { + List<String> groups = new ArrayList<>(); + groups.add("group_one"); + groups.add("group_two"); + + // Build the first response from the multiget + Map<String, Object> alertOne = new HashMap<>(); + alertOne.put(Constants.GUID, "alert_one"); + alertOne.put(THREAT_FIELD_DEFAULT, 10.0d); + + // Build the second response from the multiget + Map<String, Object> alertTwo = new HashMap<>(); + alertTwo.put(Constants.GUID, "alert_one"); + alertTwo.put(THREAT_FIELD_DEFAULT, 5.0d); + List<Document> alerts = new ArrayList<>(); + alerts.add(new Document(alertOne, "", "", 0L)); + alerts.add(new Document(alertTwo, "", "", 0L)); + + // Actually build the doc + Document actual = dao.buildCreateDocument(alerts, groups, ALERT_FIELD); + + ArrayList<Map<String, Object>> alertList = new ArrayList<>(); + alertList.add(alertOne); + alertList.add(alertTwo); + + Map<String, Object> actualDocument = actual.getDocument(); + assertNotNull(actualDocument.get(Fields.TIMESTAMP.getName())); + assertEquals( + alertList, + actualDocument.get(ALERT_FIELD) + ); + assertEquals( + groups, + actualDocument.get(GROUPS_FIELD) + ); + + // Don't care about the result, just that it's a UUID. Exception will be thrown if not. + UUID.fromString((String) actualDocument.get(Constants.GUID)); + } + + // Utility method to manage comparing update maps + protected boolean updatesMapEquals(Map<Document, Optional<String>> expected, + Map<Document, Optional<String>> actual) { + Entry<Document, Optional<String>> expectedMetaEntry; + Entry<Document, Optional<String>> actualMetaEntry; + + expectedMetaEntry = findMetaEntry(expected); + actualMetaEntry = findMetaEntry(actual); + + // Compare the metaalerts directly: they can mess with comparison because of float scores. + if (!metaAlertDocumentEquals(expectedMetaEntry.getKey(), actualMetaEntry.getKey())) { + return false; + } else { + // Remove the potentially problematic metaalert comparison. + return removeMetaEntry(expected).equals(removeMetaEntry(actual)); + } + } + + protected Entry<Document, Optional<String>> findMetaEntry( + Map<Document, Optional<String>> expected) { + for (Entry<Document, Optional<String>> entry : expected.entrySet()) { + if (entry.getKey().getSensorType().equals(METAALERT_TYPE)) { + return entry; + } + } + return null; + } + + // Unfortunately, the floating point comparison problem prevents direct remove call. + protected Map<Document, Optional<String>> removeMetaEntry( + Map<Document, Optional<String>> updates) { + Map<Document, Optional<String>> filteredUpdates = new HashMap<>(); + for (Entry<Document, Optional<String>> entry : updates.entrySet()) { + if (!(entry.getKey().getSensorType().equals(METAALERT_TYPE))) { + filteredUpdates.put(entry.getKey(), entry.getValue()); + } + } + return filteredUpdates; + } + + + // Utility method to ensure that the floating point values contained in a metaalert don't get + // incorrectly evaluated as not equal. + private boolean metaAlertDocumentEquals(Document expected, Document actual) { + if (!expected.getGuid().equals(actual.getGuid())) { + return false; + } + if (!expected.getSensorType().equals(actual.getSensorType())) { + return false; + } + if (!expected.getTimestamp().equals(actual.getTimestamp())) { + return false; + } + + // The underlying documents have to be compared more thoroughly since it has floating point + Map<String, Object> expectedDocument = expected.getDocument(); + Map<String, Object> actualDocument = actual.getDocument(); + + if (expectedDocument.size() != actualDocument.size()) { + return false; + } + + for (Entry<String, Object> entry : expectedDocument.entrySet()) { + Object value = entry.getValue(); + Object actualValue = actual.getDocument().get(entry.getKey()); + if (value instanceof Float) { + if (!MathUtils.equals((Float) value, (Float) actualValue, EPS)) { + return false; + } + } else if (value instanceof Double) { + if (!MathUtils.equals((Double) value, (Double) actualValue, EPS)) { + return false; + } + } else { + if (!value.equals(actual.getDocument().get(entry.getKey()))) { + return false; + } + } + } + + return true; + } + + // Generate some child alerts. + protected List<Document> buildChildAlerts(int num, String parent, String guidPrefix) { + String prefix = guidPrefix != null ? guidPrefix : DEFAULT_PREFIX; + List<Document> alerts = new ArrayList<>(); + for (int i = 0; i < num; i++) { + HashMap<String, Object> fields = new HashMap<>(); + fields.put(Constants.GUID, prefix + i); + fields.put(THREAT_FIELD_DEFAULT, 0.0f); + if (parent != null) { + fields.put(METAALERT_FIELD, Collections.singletonList(parent)); + } else { + fields.put(METAALERT_FIELD, new ArrayList<>()); + } + alerts.add(new Document(fields, prefix + i, "test", 0L)); + } + return alerts; + } + + protected List<Map<String, Object>> getRawMaps(List<Document> documents) { + List<Map<String, Object>> rawMaps = new ArrayList<>(); + for (Document document : documents) { + rawMaps.add(document.getDocument()); + } + return rawMaps; + } + + protected Document buildMetaAlert(List<Document> alerts) { + Map<String, Object> metaAlertMap = new HashMap<>(); + metaAlertMap.put(ALERT_FIELD, getRawMaps(alerts)); + metaAlertMap.put(Constants.GUID, METAALERT_GUID); + return new Document( + metaAlertMap, + METAALERT_GUID, + METAALERT_TYPE, + 0L + ); + } +}
http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java index 1671ab3..5cb57d7 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java @@ -52,12 +52,10 @@ public abstract class IndexingIntegrationTest extends BaseIntegrationTest { protected final int NUM_RETRIES = 100; protected final long TOTAL_TIME_MS = 150000L; - protected void preTest() {} - + protected void preTest() { } @Test public void test() throws Exception { - preTest(); final List<byte[]> inputMessages = TestUtils.readSampleData(sampleParsedPath); final Properties topologyProperties = new Properties() {{ setProperty("indexing_kafka_start", "UNCOMMITTED_EARLIEST"); http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-pcap-backend/.gitignore ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap-backend/.gitignore b/metron-platform/metron-pcap-backend/.gitignore new file mode 100644 index 0000000..df1a13b --- /dev/null +++ b/metron-platform/metron-pcap-backend/.gitignore @@ -0,0 +1 @@ +/logs \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/pom.xml b/metron-platform/metron-solr/pom.xml index 9b2e806..736fd15 100644 --- a/metron-platform/metron-solr/pom.xml +++ b/metron-platform/metron-solr/pom.xml @@ -31,7 +31,7 @@ <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> - <version>${global_hbase_guava_version}</version> + <version>${global_guava_version}</version> </dependency> <dependency> <groupId>org.apache.solr</groupId> @@ -300,7 +300,7 @@ <relocations> <relocation> <pattern>com.google.common</pattern> - <shadedPattern>org.apache.metron.guava</shadedPattern> + <shadedPattern>org.apache.metron.guava.metron-solr</shadedPattern> </relocation> <relocation> <pattern>com.fasterxml.jackson</pattern> http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml b/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml index b463366..ca69304 100644 --- a/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml +++ b/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml @@ -677,6 +677,9 @@ <dynamicField name="*.reason" type="string" multiValued="false" docValues="true"/> <dynamicField name="*.name" type="string" multiValued="false" docValues="true"/> + <!-- Metaalerts Field --> + <field name="metaalerts" type="string" multiValued="true" indexed="true" stored="true"/> + <!-- Catch all, if we don't know about it, it gets dropped. --> <dynamicField name="*" type="ignored" multiValued="false" docValues="true"/> http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/main/config/schema/metaalert/schema.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/config/schema/metaalert/schema.xml b/metron-platform/metron-solr/src/main/config/schema/metaalert/schema.xml index e36c71e..63e729b 100644 --- a/metron-platform/metron-solr/src/main/config/schema/metaalert/schema.xml +++ b/metron-platform/metron-solr/src/main/config/schema/metaalert/schema.xml @@ -15,27 +15,44 @@ See the License for the specific language governing permissions and limitations under the License. --> + <schema name="metaalert_doc" version="1.6"> <field name="_version_" type="plong" indexed="true" stored="true"/> - <field name="_root_" type="string" indexed="true" stored="false" docValues="false" /> - <field name="guid" type="string" indexed="true" stored="true" required="true" multiValued="false" /> - <field name="score" type="string" indexed="true" stored="true" /> - <field name="status" type="string" indexed="true" stored="true" /> - <field name="timestamp" type="timestamp" indexed="true" stored="true" /> - <field name="source.type" type="string" indexed="true" stored="true" /> - <dynamicField name="alert.*" type="string" multiValued="false" docValues="true"/> - <dynamicField name="*score" type="pfloat" multiValued="false" docValues="true"/> - <dynamicField name="*" type="ignored" multiValued="false" docValues="true"/> + <field name="_root_" type="string" indexed="true" stored="false" docValues="false"/> + <field name="_childDocuments_" type="ignored" stored="true" docValues="true"/> + + <field name="guid" type="string" indexed="true" stored="true" required="true" + multiValued="false"/> + + <field name="source.type" type="string" indexed="true" stored="true"/> + <field name="timestamp" type="plong" indexed="true" stored="true"/> + <field name="score" type="pdouble" indexed="true" stored="true"/> + <field name="status" type="string" indexed="true" stored="true"/> + <field name="threat:triage:score" type="pdouble" indexed="true" stored="true"/> + <field name="average" type="pdouble" indexed="true" stored="true"/> + <field name="min" type="pdouble" indexed="true" stored="true"/> + <field name="median" type="pdouble" indexed="true" stored="true"/> + <field name="max" type="pdouble" indexed="true" stored="true"/> + <field name="sum" type="pdouble" indexed="true" stored="true"/> + <field name="count" type="pint" indexed="true" stored="true"/> + <field name="groups" type="string" indexed="true" stored="true" multiValued="true"/> + + <!-- Ensure that metaalerts child field is multivalued --> + <field name="metaalerts" type="string" multiValued="true" indexed="true" stored="true"/> + + <dynamicField name="*" type="ignored" indexed="true" stored="true" multiValued="false" docValues="true"/> + <uniqueKey>guid</uniqueKey> + + <!-- Type Definitions --> <fieldType name="string" stored="true" indexed="true" multiValued="false" class="solr.StrField" sortMissingLast="true" docValues="false"/> <fieldType name="boolean" stored="true" indexed="true" multiValued="false" class="solr.BoolField" sortMissingLast="true" docValues="false"/> <fieldType name="pint" stored="true" indexed="true" multiValued="false" class="solr.TrieIntField" sortMissingLast="false" docValues="true"/> <fieldType name="pfloat" stored="true" indexed="true" multiValued="false" class="solr.TrieFloatField" sortMissingLast="false" docValues="true"/> <fieldType name="plong" stored="true" indexed="true" multiValued="false" class="solr.TrieLongField" sortMissingLast="false" docValues="true"/> <fieldType name="pdouble" stored="true" indexed="true" multiValued="false" class="solr.TrieDoubleField" sortMissingLast="false" docValues="true"/> - <fieldType name="bytes" stored="true" indexed="true" multiValued="false" class="solr.BinaryField" sortMissingLast="false" docValues="true"/> <fieldType name="location" class="solr.LatLonType" subFieldSuffix="_coordinate"/> <fieldType name="ip" stored="true" indexed="true" multiValued="false" class="solr.StrField" sortMissingLast="true" docValues="false"/> <fieldType name="timestamp" stored="true" indexed="true" multiValued="false" class="solr.TrieLongField" sortMissingLast="false" docValues="true"/> <fieldType name="ignored" stored="true" indexed="true" multiValued="true" class="solr.StrField" sortMissingLast="false" docValues="false"/> -</schema> +</schema> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml b/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml index 129c0f0..82d0320 100644 --- a/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml +++ b/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml @@ -70,6 +70,9 @@ <dynamicField name="*.reason" type="string" multiValued="false" docValues="true"/> <dynamicField name="*.name" type="string" multiValued="false" docValues="true"/> + <!-- Metaalerts Field --> + <field name="metaalerts" type="string" multiValued="true" indexed="true" stored="true"/> + <!-- Catch all, if we don't know about it, it gets dropped. --> <dynamicField name="*" type="ignored" multiValued="false" docValues="true"/> http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml b/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml index f3abb14..fc8e641 100644 --- a/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml +++ b/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml @@ -76,6 +76,9 @@ <dynamicField name="*.reason" type="string" multiValued="false" docValues="true"/> <dynamicField name="*.name" type="string" multiValued="false" docValues="true"/> + <!-- Metaalerts Field --> + <field name="metaalerts" type="string" multiValued="true" indexed="true" stored="true"/> + <!-- Catch all, if we don't know about it, it gets dropped. --> <dynamicField name="*" type="ignored" multiValued="false" docValues="true"/> http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java index b53ae20..ee541eb 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java @@ -25,6 +25,7 @@ import java.util.Optional; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.ColumnMetadataDao; import org.apache.metron.indexing.dao.IndexDao; +import org.apache.metron.indexing.dao.RetrieveLatestDao; import org.apache.metron.indexing.dao.search.FieldType; import org.apache.metron.indexing.dao.search.GetRequest; import org.apache.metron.indexing.dao.search.GroupRequest; @@ -33,6 +34,8 @@ import org.apache.metron.indexing.dao.search.InvalidSearchException; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; import org.apache.metron.indexing.dao.update.Document; +import org.apache.metron.indexing.dao.update.OriginalNotFoundException; +import org.apache.metron.indexing.dao.update.PatchRequest; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.HttpClientUtil; @@ -49,6 +52,7 @@ public class SolrDao implements IndexDao { private transient SolrClient client; private SolrSearchDao solrSearchDao; private SolrUpdateDao solrUpdateDao; + private SolrRetrieveLatestDao solrRetrieveLatestDao; private ColumnMetadataDao solrColumnMetadataDao; private AccessConfig accessConfig; @@ -57,11 +61,13 @@ public class SolrDao implements IndexDao { AccessConfig config, SolrSearchDao solrSearchDao, SolrUpdateDao solrUpdateDao, + SolrRetrieveLatestDao retrieveLatestDao, SolrColumnMetadataDao solrColumnMetadataDao) { this.client = client; this.accessConfig = config; this.solrSearchDao = solrSearchDao; this.solrUpdateDao = solrUpdateDao; + this.solrRetrieveLatestDao = retrieveLatestDao; this.solrColumnMetadataDao = solrColumnMetadataDao; } @@ -80,7 +86,8 @@ public class SolrDao implements IndexDao { this.client = getSolrClient(zkHost); this.accessConfig = config; this.solrSearchDao = new SolrSearchDao(this.client, this.accessConfig); - this.solrUpdateDao = new SolrUpdateDao(this.client); + this.solrUpdateDao = new SolrUpdateDao(this.client, this.accessConfig); + this.solrRetrieveLatestDao = new SolrRetrieveLatestDao(this.client); this.solrColumnMetadataDao = new SolrColumnMetadataDao(zkHost); } } @@ -97,12 +104,12 @@ public class SolrDao implements IndexDao { @Override public Document getLatest(String guid, String collection) throws IOException { - return this.solrSearchDao.getLatest(guid, collection); + return this.solrRetrieveLatestDao.getLatest(guid, collection); } @Override public Iterable<Document> getAllLatest(List<GetRequest> getRequests) throws IOException { - return this.solrSearchDao.getAllLatest(getRequests); + return this.solrRetrieveLatestDao.getAllLatest(getRequests); } @Override @@ -116,15 +123,35 @@ public class SolrDao implements IndexDao { } @Override + public void patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request, + Optional<Long> timestamp) + throws OriginalNotFoundException, IOException { + solrUpdateDao.patch(retrieveLatestDao, request, timestamp); + } + + @Override public Map<String, FieldType> getColumnMetadata(List<String> indices) throws IOException { return this.solrColumnMetadataDao.getColumnMetadata(indices); } - protected SolrClient getSolrClient(String zkHost) { + public SolrClient getSolrClient(String zkHost) { return new CloudSolrClient.Builder().withZkHost(zkHost).build(); } - protected void enableKerberos() { + public String getZkHost() { + Map<String, Object> globalConfig = accessConfig.getGlobalConfigSupplier().get(); + return (String) globalConfig.get("solr.zookeeper"); + } + + void enableKerberos() { HttpClientUtil.addConfigurer(new Krb5HttpClientConfigurer()); } + + public SolrSearchDao getSolrSearchDao() { + return solrSearchDao; + } + + public SolrSearchDao getSolrUpdateDao() { + return solrSearchDao; + } } http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java index 389cb4e..ca4a172 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -15,112 +15,211 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.solr.dao; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.metron.common.Constants; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.IndexDao; -import org.apache.metron.indexing.dao.MetaAlertDao; import org.apache.metron.indexing.dao.MultiIndexDao; +import org.apache.metron.indexing.dao.RetrieveLatestDao; +import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig; +import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants; import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest; import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse; +import org.apache.metron.indexing.dao.metaalert.MetaAlertDao; import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus; -import org.apache.metron.indexing.dao.search.*; +import org.apache.metron.indexing.dao.search.FieldType; +import org.apache.metron.indexing.dao.search.GetRequest; +import org.apache.metron.indexing.dao.search.GroupRequest; +import org.apache.metron.indexing.dao.search.GroupResponse; +import org.apache.metron.indexing.dao.search.InvalidCreateException; +import org.apache.metron.indexing.dao.search.InvalidSearchException; +import org.apache.metron.indexing.dao.search.SearchRequest; +import org.apache.metron.indexing.dao.search.SearchResponse; import org.apache.metron.indexing.dao.update.Document; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.Optional; +import org.apache.metron.indexing.dao.update.OriginalNotFoundException; +import org.apache.metron.indexing.dao.update.PatchRequest; +import org.apache.solr.client.solrj.SolrClient; public class SolrMetaAlertDao implements MetaAlertDao { - private SolrDao solrDao; - - @Override - public SearchResponse getAllMetaAlertsForAlert(String guid) throws InvalidSearchException { - return null; - } - - @Override - public MetaAlertCreateResponse createMetaAlert(MetaAlertCreateRequest request) throws InvalidCreateException, IOException { - return null; - } - - @Override - public boolean addAlertsToMetaAlert(String metaAlertGuid, List<GetRequest> getRequests) throws IOException { - return false; - } - - @Override - public boolean removeAlertsFromMetaAlert(String metaAlertGuid, List<GetRequest> getRequests) throws IOException { - return false; - } - - @Override - public boolean updateMetaAlertStatus(String metaAlertGuid, MetaAlertStatus status) throws IOException { - return false; - } - - @Override - public void init(IndexDao indexDao) { - - } - - @Override - public void init(IndexDao indexDao, Optional<String> threatSort) { - if (indexDao instanceof MultiIndexDao) { - MultiIndexDao multiIndexDao = (MultiIndexDao) indexDao; - for (IndexDao childDao : multiIndexDao.getIndices()) { - if (childDao instanceof SolrDao) { - this.solrDao = (SolrDao) childDao; - } - } - } else if (indexDao instanceof SolrDao) { - this.solrDao = (SolrDao) indexDao; - } else { - throw new IllegalArgumentException( - "Need an SolrDao when using SolrMetaAlertDao" - ); + public static final String METAALERTS_COLLECTION = "metaalert"; + + private IndexDao indexDao; + private SolrDao solrDao; + private SolrMetaAlertSearchDao metaAlertSearchDao; + private SolrMetaAlertUpdateDao metaAlertUpdateDao; + private SolrMetaAlertRetrieveLatestDao metaAlertRetrieveLatestDao; + protected String metaAlertsCollection = METAALERTS_COLLECTION; + protected String threatTriageField = MetaAlertConstants.THREAT_FIELD_DEFAULT; + protected String threatSort = MetaAlertConstants.THREAT_SORT_DEFAULT; + + /** + * Wraps an {@link org.apache.metron.indexing.dao.IndexDao} to handle meta alerts. + * @param indexDao The Dao to wrap + */ + public SolrMetaAlertDao(IndexDao indexDao, SolrMetaAlertSearchDao metaAlertSearchDao, + SolrMetaAlertUpdateDao metaAlertUpdateDao, + SolrMetaAlertRetrieveLatestDao metaAlertRetrieveLatestDao) { + this(indexDao, metaAlertSearchDao, metaAlertUpdateDao, metaAlertRetrieveLatestDao, + METAALERTS_COLLECTION, + MetaAlertConstants.THREAT_FIELD_DEFAULT, + MetaAlertConstants.THREAT_SORT_DEFAULT); + } + + /** + * Wraps an {@link org.apache.metron.indexing.dao.IndexDao} to handle meta alerts. + * @param indexDao The Dao to wrap + * @param triageLevelField The field name to use as the threat scoring field + * @param threatSort The summary aggregation of all child threat triage scores used + * as the overall threat triage score for the metaalert. This + * can be either max, min, average, count, median, or sum. + */ + public SolrMetaAlertDao(IndexDao indexDao, SolrMetaAlertSearchDao metaAlertSearchDao, + SolrMetaAlertUpdateDao metaAlertUpdateDao, + SolrMetaAlertRetrieveLatestDao metaAlertRetrieveLatestDao, + String metaAlertsCollection, + String triageLevelField, + String threatSort) { + init(indexDao, Optional.of(threatSort)); + this.metaAlertSearchDao = metaAlertSearchDao; + this.metaAlertUpdateDao = metaAlertUpdateDao; + this.metaAlertRetrieveLatestDao = metaAlertRetrieveLatestDao; + this.metaAlertsCollection = metaAlertsCollection; + this.threatTriageField = triageLevelField; + this.threatSort = threatSort; + } + + public SolrMetaAlertDao() { + //uninitialized. + } + + /** + * Initializes this implementation by setting the supplied IndexDao and also setting a separate SolrDao. + * This is needed for some specific Solr functions (looking up an index from a GUID for example). + * @param indexDao The DAO to wrap for our queries + * @param threatSort The summary aggregation of the child threat triage scores used + * as the overall threat triage score for the metaalert. This + * can be either max, min, average, count, median, or sum. + */ + @Override + public void init(IndexDao indexDao, Optional<String> threatSort) { + if (indexDao instanceof MultiIndexDao) { + this.indexDao = indexDao; + MultiIndexDao multiIndexDao = (MultiIndexDao) indexDao; + for (IndexDao childDao : multiIndexDao.getIndices()) { + if (childDao instanceof SolrDao) { + this.solrDao = (SolrDao) childDao; } + } + } else if (indexDao instanceof SolrDao) { + this.indexDao = indexDao; + this.solrDao = (SolrDao) indexDao; + } else { + throw new IllegalArgumentException( + "Need a SolrDao when using SolrMetaAlertDao" + ); } - @Override - public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException { - return solrDao.search(searchRequest); - } - - @Override - public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException { - return solrDao.group(groupRequest); - } - - @Override - public void init(AccessConfig config) { - - } - - @Override - public Document getLatest(String guid, String sensorType) throws IOException { - return solrDao.getLatest(guid, sensorType); - } - - @Override - public Iterable<Document> getAllLatest(List<GetRequest> getRequests) throws IOException { - return solrDao.getAllLatest(getRequests); - } - - @Override - public void update(Document update, Optional<String> index) throws IOException { - solrDao.update(update, index); - } - - @Override - public void batchUpdate(Map<Document, Optional<String>> updates) throws IOException { - solrDao.batchUpdate(updates); - } - - @Override - public Map<String, FieldType> getColumnMetadata(List<String> indices) throws IOException { - return solrDao.getColumnMetadata(indices); + MetaAlertConfig config = new MetaAlertConfig( + metaAlertsCollection, + threatTriageField, + this.threatSort, + Constants.SENSOR_TYPE + ); + + SolrClient solrClient = solrDao.getSolrClient(solrDao.getZkHost()); + this.metaAlertSearchDao = new SolrMetaAlertSearchDao(solrClient, solrDao.getSolrSearchDao()); + this.metaAlertRetrieveLatestDao = new SolrMetaAlertRetrieveLatestDao(solrDao); + this.metaAlertUpdateDao = new SolrMetaAlertUpdateDao( + solrDao, + metaAlertSearchDao, + metaAlertRetrieveLatestDao, + config); + + if (threatSort.isPresent()) { + this.threatSort = threatSort.get(); } + } + + @Override + public void init(AccessConfig config) { + // Do nothing. We're just wrapping a child dao + } + + @Override + public Map<String, FieldType> getColumnMetadata(List<String> indices) throws IOException { + return indexDao.getColumnMetadata(indices); + } + + @Override + public Document getLatest(String guid, String sensorType) throws IOException { + return metaAlertRetrieveLatestDao.getLatest(guid, sensorType); + } + + @Override + public Iterable<Document> getAllLatest(List<GetRequest> getRequests) throws IOException { + return metaAlertRetrieveLatestDao.getAllLatest(getRequests); + } + + @Override + public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException { + return metaAlertSearchDao.search(searchRequest); + } + + @Override + public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException { + return metaAlertSearchDao.group(groupRequest); + } + + @Override + public void update(Document update, Optional<String> index) throws IOException { + metaAlertUpdateDao.update(update, index); + } + + @Override + public void batchUpdate(Map<Document, Optional<String>> updates) { + metaAlertUpdateDao.batchUpdate(updates); + } + + @Override + public void patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request, + Optional<Long> timestamp) + throws OriginalNotFoundException, IOException { + metaAlertUpdateDao.patch(retrieveLatestDao, request, timestamp); + } + + @Override + public SearchResponse getAllMetaAlertsForAlert(String guid) throws InvalidSearchException { + return metaAlertSearchDao.getAllMetaAlertsForAlert(guid); + } + + @Override + public MetaAlertCreateResponse createMetaAlert(MetaAlertCreateRequest request) + throws InvalidCreateException, IOException { + return metaAlertUpdateDao.createMetaAlert(request); + } + + @Override + public boolean addAlertsToMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests) + throws IOException { + return metaAlertUpdateDao.addAlertsToMetaAlert(metaAlertGuid, alertRequests); + } + + @Override + public boolean removeAlertsFromMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests) + throws IOException { + return metaAlertUpdateDao.removeAlertsFromMetaAlert(metaAlertGuid, alertRequests); + } + + @Override + public boolean updateMetaAlertStatus(String metaAlertGuid, MetaAlertStatus status) + throws IOException { + return metaAlertUpdateDao.updateMetaAlertStatus(metaAlertGuid, status); + } } http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertRetrieveLatestDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertRetrieveLatestDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertRetrieveLatestDao.java new file mode 100644 index 0000000..7afe113 --- /dev/null +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertRetrieveLatestDao.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.solr.dao; + +import static org.apache.metron.solr.dao.SolrMetaAlertDao.METAALERTS_COLLECTION; + +import java.io.IOException; +import java.util.List; +import org.apache.metron.common.Constants; +import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants; +import org.apache.metron.indexing.dao.metaalert.MetaAlertRetrieveLatestDao; +import org.apache.metron.indexing.dao.search.GetRequest; +import org.apache.metron.indexing.dao.update.Document; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocument; + +public class SolrMetaAlertRetrieveLatestDao implements + MetaAlertRetrieveLatestDao { + + private SolrDao solrDao; + + public SolrMetaAlertRetrieveLatestDao(SolrDao solrDao) { + this.solrDao = solrDao; + } + + @Override + public Document getLatest(String guid, String sensorType) throws IOException { + if (MetaAlertConstants.METAALERT_TYPE.equals(sensorType)) { + // Unfortunately, we can't just defer to the indexDao for this. Child alerts in Solr end up + // having to be dug out. + String guidClause = Constants.GUID + ":" + guid; + SolrQuery query = new SolrQuery(); + query.setQuery(guidClause) + .setFields("*", "[child parentFilter=" + guidClause + " limit=999]"); + + try { + QueryResponse response = solrDao.getSolrClient(solrDao.getZkHost()) + .query(METAALERTS_COLLECTION, query); + // GUID is unique, so it's definitely the first result + if (response.getResults().size() == 1) { + SolrDocument result = response.getResults().get(0); + + return SolrUtilities.toDocument(result); + } else { + return null; + } + } catch (SolrServerException e) { + throw new IOException("Unable to retrieve metaalert", e); + } + } else { + return solrDao.getLatest(guid, sensorType); + } + } + + @Override + public Iterable<Document> getAllLatest(List<GetRequest> getRequests) throws IOException { + return solrDao.getAllLatest(getRequests); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java new file mode 100644 index 0000000..6b5b3a8 --- /dev/null +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.solr.dao; + +import static org.apache.metron.solr.dao.SolrMetaAlertDao.METAALERTS_COLLECTION; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.apache.metron.common.Constants; +import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants; +import org.apache.metron.indexing.dao.metaalert.MetaAlertSearchDao; +import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus; +import org.apache.metron.indexing.dao.search.GroupRequest; +import org.apache.metron.indexing.dao.search.GroupResponse; +import org.apache.metron.indexing.dao.search.InvalidSearchException; +import org.apache.metron.indexing.dao.search.SearchRequest; +import org.apache.metron.indexing.dao.search.SearchResponse; +import org.apache.metron.indexing.dao.search.SearchResult; +import org.apache.metron.indexing.dao.update.Document; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.util.ClientUtils; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.params.CursorMarkParams; +import org.apache.solr.common.params.MapSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SolrMetaAlertSearchDao implements MetaAlertSearchDao { + + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + transient SolrSearchDao solrSearchDao; + transient SolrClient solrClient; + + public SolrMetaAlertSearchDao(SolrClient solrClient, SolrSearchDao solrSearchDao) { + this.solrClient = solrClient; + this.solrSearchDao = solrSearchDao; + } + + @Override + public SearchResponse getAllMetaAlertsForAlert(String guid) throws InvalidSearchException { + if (guid == null || guid.trim().isEmpty()) { + throw new InvalidSearchException("Guid cannot be empty"); + } + + // Searches for all alerts containing the meta alert guid in it's "metalerts" array + // The query has to match the parentFilter to avoid errors. Guid must also be explicitly + // included. + String activeClause = + MetaAlertConstants.STATUS_FIELD + ":" + MetaAlertStatus.ACTIVE.getStatusString(); + String guidClause = Constants.GUID + ":" + guid; + String fullClause = "{!parent which=" + activeClause + "}" + guidClause; + String metaalertTypeClause = Constants.SENSOR_TYPE + ":" + MetaAlertConstants.METAALERT_TYPE; + SolrQuery solrQuery = new SolrQuery() + .setQuery(fullClause) + .setFields("*", "[child parentFilter=" + metaalertTypeClause + " limit=999]") + .addSort(Constants.GUID, + SolrQuery.ORDER.asc); // Just do basic sorting to track where we are + + // Use Solr's Cursors to handle the paging, rather than doing it manually. + List<SearchResult> allResults = new ArrayList<>(); + try { + String cursorMark = CursorMarkParams.CURSOR_MARK_START; + boolean done = false; + while (!done) { + solrQuery.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark); + QueryResponse rsp = solrClient.query(METAALERTS_COLLECTION, solrQuery); + String nextCursorMark = rsp.getNextCursorMark(); + rsp.getResults().stream() + .map(solrDocument -> SolrUtilities.getSearchResult(solrDocument, null)) + .forEachOrdered(allResults::add); + if (cursorMark.equals(nextCursorMark)) { + done = true; + } + cursorMark = nextCursorMark; + } + } catch (IOException | SolrServerException e) { + throw new InvalidSearchException("Unable to complete search", e); + } + + SearchResponse searchResponse = new SearchResponse(); + searchResponse.setResults(allResults); + searchResponse.setTotal(allResults.size()); + return searchResponse; + } + + @Override + public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException { + // Need to wrap such that two things are true + // 1. The provided query is true OR nested query on the alert field is true + // 2. Metaalert is active OR it's not a metaalert + + String activeStatusClause = + MetaAlertConstants.STATUS_FIELD + ":" + MetaAlertStatus.ACTIVE.getStatusString(); + + String metaalertTypeClause = Constants.SENSOR_TYPE + ":" + MetaAlertConstants.METAALERT_TYPE; + // Use the 'v=' form in order to ensure complex clauses are properly handled. + // Per the docs, the 'which=' clause should be used to identify all metaalert parents, not to + // filter + // Status is a filter on parents and must be done outside the '!parent' construct + String parentChildQuery = + "(+" + activeStatusClause + " +" + "{!parent which=" + metaalertTypeClause + " v='" + + searchRequest.getQuery() + "'})"; + + // Put everything together to get our full query + // The '-metaalert:[* TO *]' construct is to ensure the field doesn't exist on or is empty for + // plain alerts. + // Also make sure that it's not a metaalert + String fullQuery = + "(" + searchRequest.getQuery() + " AND -" + MetaAlertConstants.METAALERT_FIELD + ":[* TO *]" + + " AND " + "-" + metaalertTypeClause + ")" + " OR " + parentChildQuery; + + LOG.debug("MetaAlert search query {}", fullQuery); + + searchRequest.setQuery(fullQuery); + + // Build the custom field list + List<String> fields = searchRequest.getFields(); + String fieldList = "*"; + if (fields != null) { + fieldList = StringUtils.join(fields, ","); + } + + LOG.debug("MetaAlert Search Field list {}", fullQuery); + + SearchResponse results = solrSearchDao.search(searchRequest, fieldList); + LOG.debug("MetaAlert Search Number of results {}", results.getResults().size()); + + // Unfortunately, we can't get the full metaalert results at the same time + // Get them in a second query. + // However, we can only retrieve them if we have the source type field (either explicit or + // wildcard). + if (fieldList.contains("*") || fieldList.contains(Constants.SENSOR_TYPE)) { + List<String> metaalertGuids = new ArrayList<>(); + for (SearchResult result : results.getResults()) { + if (result.getSource().get(Constants.SENSOR_TYPE) + .equals(MetaAlertConstants.METAALERT_TYPE)) { + // Then we need to add it to the list to retrieve child alerts in a second query. + metaalertGuids.add(result.getId()); + } + } + LOG.debug("MetaAlert Search guids requiring retrieval: {}", metaalertGuids); + + // If we have any metaalerts in our result, attach the full data. + if (metaalertGuids.size() > 0) { + Map<String, String> params = new HashMap<>(); + params.put("fl", fieldList + ",[child parentFilter=" + metaalertTypeClause + " limit=999]"); + SolrParams solrParams = new MapSolrParams(params); + try { + SolrDocumentList solrDocumentList = solrClient + .getById(METAALERTS_COLLECTION, metaalertGuids, solrParams); + Map<String, Document> guidToDocuments = new HashMap<>(); + for (SolrDocument doc : solrDocumentList) { + Document document = SolrUtilities.toDocument(doc); + guidToDocuments.put(document.getGuid(), document); + } + + // Run through our results and update them with the full metaalert + for (SearchResult result : results.getResults()) { + Document fullDoc = guidToDocuments.get(result.getId()); + if (fullDoc != null) { + result.setSource(fullDoc.getDocument()); + } + } + } catch (SolrServerException | IOException e) { + throw new InvalidSearchException("Error when retrieving child alerts for metaalerts", e); + } + + } + } + return results; + } + + @Override + public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException { + // Make sure to escape any problematic characters here + String sourceType = ClientUtils.escapeQueryChars(Constants.SENSOR_TYPE); + String baseQuery = groupRequest.getQuery(); + String adjustedQuery = baseQuery + " -" + MetaAlertConstants.METAALERT_FIELD + ":[* TO *]" + + " -" + sourceType + ":" + MetaAlertConstants.METAALERT_TYPE; + LOG.debug("MetaAlert group adjusted query: {}", adjustedQuery); + groupRequest.setQuery(adjustedQuery); + return solrSearchDao.group(groupRequest); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java new file mode 100644 index 0000000..b00954a --- /dev/null +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.solr.dao; + +import static org.apache.metron.solr.dao.SolrMetaAlertDao.METAALERTS_COLLECTION; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.metron.common.Constants; +import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig; +import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants; +import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest; +import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse; +import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus; +import org.apache.metron.indexing.dao.metaalert.MetaAlertUpdateDao; +import org.apache.metron.indexing.dao.metaalert.MetaScores; +import org.apache.metron.indexing.dao.metaalert.lucene.AbstractLuceneMetaAlertUpdateDao; +import org.apache.metron.indexing.dao.search.GetRequest; +import org.apache.metron.indexing.dao.search.InvalidCreateException; +import org.apache.metron.indexing.dao.search.InvalidSearchException; +import org.apache.metron.indexing.dao.search.SearchResponse; +import org.apache.metron.indexing.dao.search.SearchResult; +import org.apache.metron.indexing.dao.update.Document; +import org.apache.metron.indexing.dao.update.UpdateDao; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrServerException; + +public class SolrMetaAlertUpdateDao extends AbstractLuceneMetaAlertUpdateDao implements + MetaAlertUpdateDao, UpdateDao { + + private SolrClient solrClient; + private SolrMetaAlertSearchDao metaAlertSearchDao; + + /** + * Constructor a SolrMetaAlertUpdateDao + * @param solrDao An SolrDao to defer queries to. + * @param metaAlertSearchDao A MetaAlert aware search DAO used in retrieving items being mutated. + * @param retrieveLatestDao A RetrieveLatestDao for getting the current state of items being + * mutated. + */ + public SolrMetaAlertUpdateDao(SolrDao solrDao, + SolrMetaAlertSearchDao metaAlertSearchDao, + SolrMetaAlertRetrieveLatestDao retrieveLatestDao, + MetaAlertConfig config) { + super(solrDao, retrieveLatestDao, config); + this.solrClient = solrDao.getSolrClient(solrDao.getZkHost()); + this.metaAlertSearchDao = metaAlertSearchDao; + } + + @Override + public MetaAlertCreateResponse createMetaAlert(MetaAlertCreateRequest request) + throws InvalidCreateException, IOException { + List<GetRequest> alertRequests = request.getAlerts(); + if (request.getAlerts().isEmpty()) { + throw new InvalidCreateException("MetaAlertCreateRequest must contain alerts"); + } + if (request.getGroups().isEmpty()) { + throw new InvalidCreateException("MetaAlertCreateRequest must contain UI groups"); + } + + // Retrieve the documents going into the meta alert and build it + Iterable<Document> alerts = getRetrieveLatestDao().getAllLatest(alertRequests); + + Document metaAlert = buildCreateDocument(alerts, request.getGroups(), + MetaAlertConstants.ALERT_FIELD); + MetaScores.calculateMetaScores(metaAlert, getConfig().getThreatTriageField(), + getConfig().getThreatSort()); + + // Add source type to be consistent with other sources and allow filtering + metaAlert.getDocument().put(Constants.SENSOR_TYPE, MetaAlertConstants.METAALERT_TYPE); + + // Start a list of updates / inserts we need to run + Map<Document, Optional<String>> updates = new HashMap<>(); + updates.put(metaAlert, Optional.of(METAALERTS_COLLECTION)); + + try { + // We need to update the associated alerts with the new meta alerts, making sure existing + // links are maintained. + Map<String, Optional<String>> guidToIndices = alertRequests.stream().collect(Collectors.toMap( + GetRequest::getGuid, GetRequest::getIndex)); + Map<String, String> guidToSensorTypes = alertRequests.stream().collect(Collectors.toMap( + GetRequest::getGuid, GetRequest::getSensorType)); + for (Document alert : alerts) { + if (addMetaAlertToAlert(metaAlert.getGuid(), alert)) { + // Use the index in the request if it exists + Optional<String> index = guidToIndices.get(alert.getGuid()); + if (!index.isPresent()) { + index = Optional.ofNullable(guidToSensorTypes.get(alert.getGuid())); + if (!index.isPresent()) { + throw new IllegalArgumentException("Could not find index for " + alert.getGuid()); + } + } + updates.put(alert, index); + } + } + + // Kick off any updates. + update(updates); + + MetaAlertCreateResponse createResponse = new MetaAlertCreateResponse(); + createResponse.setCreated(true); + createResponse.setGuid(metaAlert.getGuid()); + solrClient.commit(METAALERTS_COLLECTION); + return createResponse; + } catch (IOException | SolrServerException e) { + throw new InvalidCreateException("Unable to create meta alert", e); + } + } + + + /** + * Updates a document in Solr for a given collection. Collection is not optional for Solr. + * @param update The update to be run + * @param collection The index to be updated. Mandatory for Solr + * @throws IOException Thrown when an error occurs during the write. + */ + @Override + public void update(Document update, Optional<String> collection) throws IOException { + if (MetaAlertConstants.METAALERT_TYPE.equals(update.getSensorType())) { + // We've been passed an update to the meta alert. + throw new UnsupportedOperationException("Meta alerts cannot be directly updated"); + } + // Index can't be optional, or it won't be committed + + Map<Document, Optional<String>> updates = new HashMap<>(); + updates.put(update, collection); + + // We need to update an alert itself. It cannot be delegated in Solr; we need to retrieve all + // metaalerts and update the entire document for each. + SearchResponse searchResponse; + try { + searchResponse = metaAlertSearchDao.getAllMetaAlertsForAlert(update.getGuid()); + } catch (InvalidSearchException e) { + throw new IOException("Unable to retrieve metaalerts for alert", e); + } + + ArrayList<Document> metaAlerts = new ArrayList<>(); + for (SearchResult searchResult : searchResponse.getResults()) { + Document doc = new Document(searchResult.getSource(), searchResult.getId(), + MetaAlertConstants.METAALERT_TYPE, 0L); + metaAlerts.add(doc); + } + + for (Document metaAlert : metaAlerts) { + if (replaceAlertInMetaAlert(metaAlert, update)) { + updates.put(metaAlert, Optional.of(METAALERTS_COLLECTION)); + } + } + + // Run the alert's update + getUpdateDao().batchUpdate(updates); + + try { + solrClient.commit(METAALERTS_COLLECTION); + if (collection.isPresent()) { + solrClient.commit(collection.get()); + } + } catch (SolrServerException e) { + throw new IOException("Unable to update document", e); + } + } + + protected boolean replaceAlertInMetaAlert(Document metaAlert, Document alert) { + boolean metaAlertUpdated = removeAlertsFromMetaAlert(metaAlert, + Collections.singleton(alert.getGuid())); + if (metaAlertUpdated) { + addAlertsToMetaAlert(metaAlert, Collections.singleton(alert)); + } + return metaAlertUpdated; + } + + @Override + public boolean addAlertsToMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests) + throws IOException { + boolean success; + Document metaAlert = getRetrieveLatestDao() + .getLatest(metaAlertGuid, MetaAlertConstants.METAALERT_TYPE); + if (MetaAlertStatus.ACTIVE.getStatusString() + .equals(metaAlert.getDocument().get(MetaAlertConstants.STATUS_FIELD))) { + Iterable<Document> alerts = getRetrieveLatestDao().getAllLatest(alertRequests); + Map<Document, Optional<String>> updates = buildAddAlertToMetaAlertUpdates(metaAlert, alerts); + update(updates); + success = updates.size() != 0; + } else { + throw new IllegalStateException("Adding alerts to an INACTIVE meta alert is not allowed"); + } + try { + solrClient.commit(METAALERTS_COLLECTION); + } catch (SolrServerException e) { + throw new IOException("Unable to commit alerts to metaalert: " + metaAlertGuid, e); + } + return success; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java new file mode 100644 index 0000000..8578bfb --- /dev/null +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.solr.dao; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.metron.indexing.dao.RetrieveLatestDao; +import org.apache.metron.indexing.dao.search.GetRequest; +import org.apache.metron.indexing.dao.update.Document; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; + +public class SolrRetrieveLatestDao implements RetrieveLatestDao { + + private transient SolrClient client; + + public SolrRetrieveLatestDao(SolrClient client) { + this.client = client; + } + + @Override + public Document getLatest(String guid, String collection) throws IOException { + try { + SolrDocument solrDocument = client.getById(collection, guid); + if (solrDocument == null) { + return null; + } + return SolrUtilities.toDocument(solrDocument); + } catch (SolrServerException e) { + throw new IOException(e); + } + } + + @Override + public Iterable<Document> getAllLatest(List<GetRequest> getRequests) throws IOException { + Map<String, Collection<String>> collectionIdMap = new HashMap<>(); + for (GetRequest getRequest : getRequests) { + Collection<String> ids = collectionIdMap + .getOrDefault(getRequest.getSensorType(), new HashSet<>()); + ids.add(getRequest.getGuid()); + collectionIdMap.put(getRequest.getSensorType(), ids); + } + try { + List<Document> documents = new ArrayList<>(); + for (String collection : collectionIdMap.keySet()) { + SolrDocumentList solrDocumentList = client.getById(collectionIdMap.get(collection), + new SolrQuery().set("collection", collection)); + documents.addAll( + solrDocumentList.stream().map(SolrUtilities::toDocument).collect(Collectors.toList())); + } + return documents; + } catch (SolrServerException e) { + throw new IOException(e); + } + } +}