METRON-1577 Solr searches don't include the index of the result (merrimanr) closes apache/metron#1031
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/7a071f6d Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/7a071f6d Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/7a071f6d Branch: refs/heads/feature/METRON-1554-pcap-query-panel Commit: 7a071f6d64794267f40a0ef1810bd7deaf044eeb Parents: 49f851e Author: merrimanr <merrim...@gmail.com> Authored: Fri May 25 08:41:44 2018 -0500 Committer: merrimanr <merrim...@gmail.com> Committed: Fri May 25 08:41:44 2018 -0500 ---------------------------------------------------------------------- .../src/app/service/alerts.service.ts | 2 +- .../apache/metron/rest/config/IndexConfig.java | 3 +- .../metron-rest/src/main/scripts/metron-rest.sh | 3 + .../ElasticsearchSearchIntegrationTest.java | 9 +++ .../metron/indexing/util/IndexingCacheUtil.java | 14 ++++- .../indexing/dao/SearchIntegrationTest.java | 4 ++ .../indexing/util/IndexingCacheUtilTest.java | 66 ++++++++++++++++++++ .../org/apache/metron/solr/SolrConstants.java | 1 + .../metron/solr/dao/SolrMetaAlertSearchDao.java | 3 +- .../apache/metron/solr/dao/SolrSearchDao.java | 7 ++- .../apache/metron/solr/dao/SolrUpdateDao.java | 23 +++++-- .../apache/metron/solr/dao/SolrUtilities.java | 5 +- .../apache/metron/solr/writer/SolrWriter.java | 4 +- .../metron/solr/dao/SolrSearchDaoTest.java | 8 ++- .../metron/solr/dao/SolrUpdateDaoTest.java | 34 ++++++++-- .../integration/SolrSearchIntegrationTest.java | 6 ++ 16 files changed, 172 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-interface/metron-alerts/src/app/service/alerts.service.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/src/app/service/alerts.service.ts b/metron-interface/metron-alerts/src/app/service/alerts.service.ts index 7324a72..56939d8 100644 --- a/metron-interface/metron-alerts/src/app/service/alerts.service.ts +++ b/metron-interface/metron-alerts/src/app/service/alerts.service.ts @@ -30,7 +30,7 @@ export class AlertsService { } public escalate(alerts: Alert[]): Observable<null> { - return this.http.post('/api/v1/alert/escalate', alerts, new RequestOptions({headers: new Headers(this.defaultHeaders)})) + return this.http.post('/api/v1/alerts/ui/escalate', alerts, new RequestOptions({headers: new Headers(this.defaultHeaders)})) .catch(HttpUtil.handleError); } } http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java index c432c6c..53b10f9 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java @@ -18,6 +18,7 @@ package org.apache.metron.rest.config; import static org.apache.metron.rest.MetronRestConstants.INDEX_DAO_IMPL; +import static org.apache.metron.rest.MetronRestConstants.INDEX_WRITER_NAME; import java.util.Optional; import org.apache.metron.common.zookeeper.ConfigurationsCache; @@ -73,7 +74,7 @@ public class IndexConfig { throw new IllegalStateException("Unable to retrieve the global config.", e); } }); - config.setIndexSupplier(IndexingCacheUtil.getIndexLookupFunction(cache)); + config.setIndexSupplier(IndexingCacheUtil.getIndexLookupFunction(cache, environment.getProperty(INDEX_WRITER_NAME))); config.setTableProvider(TableProvider.create(hbaseProviderImpl, () -> new HTableProvider())); config.setKerberosEnabled(environment.getProperty(MetronRestConstants.KERBEROS_ENABLED_SPRING_PROPERTY, Boolean.class, false)); if (indexDaoImpl == null) { http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-interface/metron-rest/src/main/scripts/metron-rest.sh ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh index 20b535e..21e8128 100644 --- a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh +++ b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh @@ -113,10 +113,13 @@ echo "METRON_REST_CLASSPATH=${METRON_REST_CLASSPATH}" if [[ ${METRON_RA_INDEXING_WRITER} == "Solr" ]]; then METRON_INDEX_DAO=" --index.dao.impl=org.apache.metron.solr.dao.SolrDao,org.apache.metron.indexing.dao.HBaseDao" METRON_METAALERT_DAO=" --meta.dao.impl=org.apache.metron.solr.dao.SolrMetaAlertDao" + METRON_WRITER_NAME=" --index.writer.name=solr" echo "METRON_INDEX_DAO=${METRON_INDEX_DAO}" echo "METRON_METAALERT_DAO=${METRON_METAALERT_DAO}" + echo "METRON_WRITER_NAME=${METRON_WRITER_NAME}" METRON_SPRING_OPTIONS+=${METRON_INDEX_DAO} METRON_SPRING_OPTIONS+=${METRON_METAALERT_DAO} + METRON_SPRING_OPTIONS+=${METRON_WRITER_NAME} fi echo "Starting application" http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java index 6f76093..8071e68 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java @@ -361,4 +361,13 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { protected IndexDao getIndexDao() { return dao; } + + @Override + protected String getIndexName(String sensorType) { + if ("bro".equals(sensorType)) { + return "bro_index_2017.01.01.01"; + } else { + return "snort_index_2017.01.01.02"; + } + } } http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/util/IndexingCacheUtil.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/util/IndexingCacheUtil.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/util/IndexingCacheUtil.java index 3ff3a20..86a0642 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/util/IndexingCacheUtil.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/util/IndexingCacheUtil.java @@ -24,12 +24,20 @@ import org.apache.metron.common.configuration.IndexingConfigurations; import org.apache.metron.common.zookeeper.ConfigurationsCache; public class IndexingCacheUtil { - public static Function<String, String> getIndexLookupFunction(ConfigurationsCache cache) { + + @SuppressWarnings("unchecked") + public static Function<String, String> getIndexLookupFunction(ConfigurationsCache cache, String writerName) { return sensorType -> { + String indexingTopic = sensorType; IndexingConfigurations indexingConfigs = cache.get( IndexingConfigurations.class); Map<String, Object> indexingSensorConfigs = indexingConfigs.getSensorIndexingConfig(sensorType); - String indexingTopic = (String) indexingSensorConfigs.get(IndexingConfigurations.INDEX_CONF); - return indexingTopic != null ? indexingTopic : sensorType; + if (indexingSensorConfigs != null) { + Map<String, Object> writerConfigs = (Map<String, Object>) indexingSensorConfigs.get(writerName); + if (writerConfigs != null) { + indexingTopic = (String) writerConfigs.getOrDefault(IndexingConfigurations.INDEX_CONF, indexingTopic); + } + } + return indexingTopic; }; } } http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java index 7fca764..2e1968a 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java @@ -484,10 +484,12 @@ public abstract class SearchIntegrationTest { Assert.assertEquals(10, results.size()); for(int i = 0;i < 5;++i) { Assert.assertEquals("snort", results.get(i).getSource().get(getSourceTypeField())); + Assert.assertEquals(getIndexName("snort"), results.get(i).getIndex()); Assert.assertEquals(10 - i + "", results.get(i).getSource().get("timestamp").toString()); } for (int i = 5; i < 10; ++i) { Assert.assertEquals("bro", results.get(i).getSource().get(getSourceTypeField())); + Assert.assertEquals(getIndexName("bro"), results.get(i).getIndex()); Assert.assertEquals(10 - i + "", results.get(i).getSource().get("timestamp").toString()); } } @@ -940,4 +942,6 @@ public abstract class SearchIntegrationTest { protected abstract IndexDao getIndexDao(); protected abstract String getSourceTypeField(); + + protected abstract String getIndexName(String sensorType); } http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/util/IndexingCacheUtilTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/util/IndexingCacheUtilTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/util/IndexingCacheUtilTest.java new file mode 100644 index 0000000..3d6ee85 --- /dev/null +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/util/IndexingCacheUtilTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.indexing.util; + +import org.apache.metron.common.configuration.IndexingConfigurations; +import org.apache.metron.common.zookeeper.ConfigurationsCache; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class IndexingCacheUtilTest { + + @Test + public void getIndexLookupFunctionShouldReturnConfiguredIndex() { + IndexingConfigurations indexingConfigs = mock(IndexingConfigurations.class); + ConfigurationsCache cache = mock(ConfigurationsCache.class); + + Map<String, Object> broIndexingConfig = new HashMap<String, Object>() {{ + put("writer", new HashMap<String, Object>() {{ + put("index", "bro_index"); + }}); + }}; + when(indexingConfigs.getSensorIndexingConfig("bro")).thenReturn(broIndexingConfig); + when(cache.get(IndexingConfigurations.class)).thenReturn(indexingConfigs); + + assertEquals("bro_index", IndexingCacheUtil.getIndexLookupFunction(cache, "writer").apply("bro")); + } + + @Test + public void getIndexLookupFunctionShouldDefaultToSensorType() { + IndexingConfigurations indexingConfigs = mock(IndexingConfigurations.class); + ConfigurationsCache cache = mock(ConfigurationsCache.class); + + Map<String, Object> broIndexingConfig = new HashMap<String, Object>() {{ + put("writer", new HashMap<String, Object>() {{ + put("index", "bro_index"); + }}); + }}; + when(indexingConfigs.getSensorIndexingConfig("bro")).thenReturn(broIndexingConfig); + when(cache.get(IndexingConfigurations.class)).thenReturn(indexingConfigs); + + assertEquals("Should default to sensor type on missing sensor config", "snort", IndexingCacheUtil.getIndexLookupFunction(cache, "writer").apply("snort")); + assertEquals("Should default to sensor type on missing writer config", "bro", IndexingCacheUtil.getIndexLookupFunction(cache, "someWriter").apply("bro")); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/SolrConstants.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/SolrConstants.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/SolrConstants.java index d5dc7a0..879b983 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/SolrConstants.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/SolrConstants.java @@ -26,4 +26,5 @@ public class SolrConstants { public static final String REQUEST_COLLECTION_CONFIG_NAME = "collection.configName"; public static final String REQUEST_COLLECTIONS_PATH = "/admin/collections"; public static final String RESPONSE_COLLECTIONS = "collections"; + public static final String SOLR_WRITER_NAME = "solr"; } http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java index 6b5b3a8..c6f7124 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java @@ -93,7 +93,8 @@ public class SolrMetaAlertSearchDao implements MetaAlertSearchDao { QueryResponse rsp = solrClient.query(METAALERTS_COLLECTION, solrQuery); String nextCursorMark = rsp.getNextCursorMark(); rsp.getResults().stream() - .map(solrDocument -> SolrUtilities.getSearchResult(solrDocument, null)) + .map(solrDocument -> SolrUtilities.getSearchResult(solrDocument, null, + solrSearchDao.getAccessConfig().getIndexSupplier())) .forEachOrdered(allResults::add); if (cursorMark.equals(nextCursorMark)) { done = true; http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java index 53ac083..4a8d482 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java @@ -68,6 +68,10 @@ public class SolrSearchDao implements SearchDao { this.accessConfig = accessConfig; } + protected AccessConfig getAccessConfig() { + return accessConfig; + } + @Override public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException { return search(searchRequest, null); @@ -181,7 +185,8 @@ public class SolrSearchDao implements SearchDao { // search hits --> search results List<SearchResult> results = solrDocumentList.stream() - .map(solrDocument -> SolrUtilities.getSearchResult(solrDocument, searchRequest.getFields())) + .map(solrDocument -> SolrUtilities.getSearchResult(solrDocument, searchRequest.getFields(), + accessConfig.getIndexSupplier())) .collect(Collectors.toList()); searchResponse.setResults(results); http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java index 54b10ad..7c56169 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java @@ -27,6 +27,8 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Optional; import java.util.Set; +import java.util.function.Function; + import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.update.UpdateDao; @@ -42,22 +44,33 @@ public class SolrUpdateDao implements UpdateDao { private transient SolrClient client; private AccessConfig config; - + private Function<String, String> indexSupplier; public SolrUpdateDao(SolrClient client, AccessConfig config) { this.client = client; this.config = config; + this.indexSupplier = config.getIndexSupplier(); + } + + private Optional<String> getIndex(String sensorName, Optional<String> index) { + if(index.isPresent()) { + return Optional.ofNullable(index.get()); + } + else { + String realIndex = indexSupplier.apply(sensorName); + return Optional.ofNullable(realIndex); + } } @Override - public void update(Document update, Optional<String> index) throws IOException { + public void update(Document update, Optional<String> rawIndex) throws IOException { try { SolrInputDocument solrInputDocument = SolrUtilities.toSolrInputDocument(update); + Optional<String> index = getIndex(update.getSensorType(), rawIndex); if (index.isPresent()) { this.client.add(index.get(), solrInputDocument); this.client.commit(index.get()); } else { - this.client.add(solrInputDocument); - this.client.commit(); + throw new IllegalStateException("Index must be specified or inferred."); } } catch (SolrServerException e) { throw new IOException(e); @@ -72,7 +85,7 @@ public class SolrUpdateDao implements UpdateDao { for (Entry<Document, Optional<String>> entry : updates.entrySet()) { SolrInputDocument solrInputDocument = SolrUtilities.toSolrInputDocument(entry.getKey()); - Optional<String> index = entry.getValue(); + Optional<String> index = getIndex(entry.getKey().getSensorType(), entry.getValue()); if (index.isPresent()) { Collection<SolrInputDocument> solrInputDocuments = solrCollectionUpdates .getOrDefault(index.get(), new ArrayList<>()); http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java index ce0ae84..616bd1a 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java @@ -22,6 +22,8 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; + import org.apache.metron.common.Constants; import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants; import org.apache.metron.indexing.dao.search.SearchResult; @@ -31,9 +33,10 @@ import org.apache.solr.common.SolrInputDocument; public class SolrUtilities { - public static SearchResult getSearchResult(SolrDocument solrDocument, List<String> fields) { + public static SearchResult getSearchResult(SolrDocument solrDocument, List<String> fields, Function<String, String> indexSupplier) { SearchResult searchResult = new SearchResult(); searchResult.setId((String) solrDocument.getFieldValue(Constants.GUID)); + searchResult.setIndex(indexSupplier.apply((String) solrDocument.getFieldValue(Constants.SENSOR_TYPE))); Map<String, Object> docSource = toDocument(solrDocument).getDocument(); final Map<String, Object> source = new HashMap<>(); if (fields != null) { http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/writer/SolrWriter.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/writer/SolrWriter.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/writer/SolrWriter.java index 6147966..93bd8fe 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/writer/SolrWriter.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/writer/SolrWriter.java @@ -50,6 +50,8 @@ import java.util.Set; import java.util.UUID; import java.util.function.Supplier; +import static org.apache.metron.solr.SolrConstants.SOLR_WRITER_NAME; + public class SolrWriter implements BulkMessageWriter<JSONObject>, Serializable { public static final String JAVA_SECURITY_CONFIG_PROPERTY = "java.security.auth.login.config"; @@ -237,7 +239,7 @@ public class SolrWriter implements BulkMessageWriter<JSONObject>, Serializable { @Override public String getName() { - return "solr"; + return SOLR_WRITER_NAME; } @Override http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java index 49a40ef..d17bf72 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java @@ -90,6 +90,7 @@ public class SolrSearchDaoTest { public void setUp() throws Exception { client = mock(SolrClient.class); accessConfig = mock(AccessConfig.class); + when(accessConfig.getIndexSupplier()).thenReturn(sensorType -> sensorType); solrSearchDao = new SolrSearchDao(client, accessConfig); solrRetrieveLatestDao = new SolrRetrieveLatestDao(client); mockStatic(CollectionAdminRequest.class); @@ -312,28 +313,31 @@ public class SolrSearchDaoTest { SolrDocument solrDocument = mock(SolrDocument.class); when(solrDocument.getFieldValue(Constants.GUID)).thenReturn("guid"); + when(solrDocument.getFieldValue(Constants.SENSOR_TYPE)).thenReturn("sensorType"); when(solrDocument.getFieldValue("field1")).thenReturn("value1"); when(solrDocument.getFieldValue("field2")).thenReturn("value2"); when(solrDocument.getFieldNames()).thenReturn(Arrays.asList("field1", "field2")); SearchResult expectedSearchResult = new SearchResult(); expectedSearchResult.setId("guid"); + expectedSearchResult.setIndex("sensorType"); expectedSearchResult.setSource(new HashMap<String, Object>() {{ put("field1", "value1"); }}); assertEquals(expectedSearchResult, SolrUtilities.getSearchResult(solrDocument, - Collections.singletonList("field1"))); + Collections.singletonList("field1"), solrSearchDao.getAccessConfig().getIndexSupplier())); SearchResult expectedSearchResultAllFields = new SearchResult(); expectedSearchResultAllFields.setId("guid"); + expectedSearchResultAllFields.setIndex("sensorType"); expectedSearchResultAllFields.setSource(new HashMap<String, Object>() {{ put("field1", "value1"); put("field2", "value2"); }}); assertEquals(expectedSearchResultAllFields, - SolrUtilities.getSearchResult(solrDocument, null)); + SolrUtilities.getSearchResult(solrDocument, null, solrSearchDao.getAccessConfig().getIndexSupplier())); } @Test http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java index b9ed0b5..0011aa8 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java @@ -17,8 +17,12 @@ */ package org.apache.metron.solr.dao; +import org.apache.metron.common.configuration.Configurations; +import org.apache.metron.common.configuration.IndexingConfigurations; +import org.apache.metron.common.zookeeper.ConfigurationsCache; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.update.Document; +import org.apache.metron.indexing.util.IndexingCacheUtil; import org.apache.metron.solr.matcher.SolrInputDocumentListMatcher; import org.apache.metron.solr.matcher.SolrInputDocumentMatcher; import org.apache.solr.client.solrj.SolrClient; @@ -43,6 +47,7 @@ import static org.mockito.Matchers.argThat; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; @RunWith(PowerMockRunner.class) @PrepareForTest({CollectionAdminRequest.class}) @@ -63,7 +68,17 @@ public class SolrUpdateDaoTest { put("solr.zookeeper", "zookeeper:2181"); }} ); - accessConfig.setIndexSupplier(s -> null); + IndexingConfigurations indexingConfigs = mock(IndexingConfigurations.class); + ConfigurationsCache cache = mock(ConfigurationsCache.class); + + Map<String, Object> broIndexingConfig = new HashMap<String, Object>() {{ + put("solr", new HashMap<String, Object>() {{ + }}); + }}; + when(indexingConfigs.getSensorIndexingConfig("bro")).thenReturn(broIndexingConfig); + when(cache.get(IndexingConfigurations.class)).thenReturn(indexingConfigs); + + accessConfig.setIndexSupplier(IndexingCacheUtil.getIndexLookupFunction(cache, "solr")); } @SuppressWarnings("unchecked") @@ -74,7 +89,7 @@ public class SolrUpdateDaoTest { } @Test - public void updateShouldProperlyUpdateDocument() throws Exception { + public void updateShouldProperlyUpdateDocumentImplicitIndex() throws Exception { Document document = new Document(new HashMap<String, Object>(){{ put("field", "value"); }}, "guid", "bro", 0L); @@ -84,7 +99,18 @@ public class SolrUpdateDaoTest { solrUpdateDao.update(document, Optional.empty()); - verify(client).add(argThat(new SolrInputDocumentMatcher(solrInputDocument))); + verify(client).add(eq("bro"), argThat(new SolrInputDocumentMatcher(solrInputDocument))); + + } + + @Test + public void updateShouldProperlyUpdateDocumentExplicitIndex() throws Exception { + Document document = new Document(new HashMap<String, Object>(){{ + put("field", "value"); + }}, "guid", "bro", 0L); + + SolrInputDocument solrInputDocument = new SolrInputDocument(); + solrInputDocument.addField("field", "value"); solrUpdateDao.update(document, Optional.of("bro")); @@ -144,7 +170,7 @@ public class SolrUpdateDaoTest { solrUpdateDao.batchUpdate(updates); - verify(client).add((String) Matchers.isNull(), argThat(new SolrInputDocumentListMatcher(Arrays.asList(snortSolrInputDocument1, snortSolrInputDocument2)))); + verify(client).add(eq("snort"), argThat(new SolrInputDocumentListMatcher(Arrays.asList(snortSolrInputDocument1, snortSolrInputDocument2)))); } } http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java index 2aa65ae..2fb3086 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java @@ -82,6 +82,7 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest { }} ); + config.setIndexSupplier( sensorType -> sensorType); IndexDao dao = new SolrDao(); dao.init(config); return dao; @@ -231,4 +232,9 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest { protected String getSourceTypeField() { return Constants.SENSOR_TYPE; } + + @Override + protected String getIndexName(String sensorType) { + return sensorType; + } }