[22/50] [abbrv] metron git commit: Merge branch 'master' into feature/METRON-1416-upgrade-solr
Merge branch 'master' into feature/METRON-1416-upgrade-solr
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e7233b0f
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e7233b0f
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e7233b0f
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: e7233b0f82f372e6b1339de56a115fa7e4581f07
Parents: 4170887
Author: cstella
Authored: Mon Jun 11 21:41:06 2018 -0400
Committer: cstella
Committed: Tue Jun 12 09:46:46 2018 -0400
--
.../org/apache/metron/common/Constants.java | 1 +
.../elasticsearch/dao/ElasticsearchDao.java | 4 ++
.../dao/ElasticsearchMetaAlertDao.java | 30 +-
.../dao/ElasticsearchMetaAlertUpdateDao.java| 4 +-
.../indexing/dao/metaalert/MetaAlertConfig.java | 43
.../AbstractLuceneMetaAlertUpdateDao.java | 3 ++
.../dao/metaalert/MetaAlertIntegrationTest.java | 2 +-
.../AbstractLuceneMetaAlertUpdateDaoTest.java | 28 +
.../metron/solr/dao/SolrMetaAlertDao.java | 29 +
.../metron/solr/dao/SolrMetaAlertSearchDao.java | 15 ---
.../metron/solr/dao/SolrMetaAlertUpdateDao.java | 2 +-
.../SolrMetaAlertIntegrationTest.java | 28 +
12 files changed, 129 insertions(+), 60 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/e7233b0f/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
--
diff --git
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
index f74660c..4a8bea2 100644
---
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
+++
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
@@ -28,6 +28,7 @@ public class Constants {
public static final long DEFAULT_CONFIGURED_BOLT_TIMEOUT = 5000;
public static final String SENSOR_TYPE = "source.type";
public static final String SENSOR_TYPE_FIELD_PROPERTY = "source.type.field";
+ public static final String THREAT_SCORE_FIELD_PROPERTY =
"threat.triage.score.field";
public static final String ENRICHMENT_TOPIC = "enrichments";
public static final String INDEXING_TOPIC = "indexing";
public static final String ERROR_STREAM = "error";
http://git-wip-us.apache.org/repos/asf/metron/blob/e7233b0f/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
--
diff --git
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
index eae0a39..3eb86ce 100644
---
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
+++
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
@@ -85,6 +85,10 @@ public class ElasticsearchDao implements IndexDao {
//uninitialized.
}
+ public AccessConfig getAccessConfig() {
+return accessConfig;
+ }
+
@Override
public synchronized void init(AccessConfig config) {
if (this.client == null) {
http://git-wip-us.apache.org/repos/asf/metron/blob/e7233b0f/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
--
diff --git
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
index f73a640..649077e 100644
---
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
+++
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
@@ -68,6 +68,7 @@ import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
+import java.util.function.Supplier;
import java.util.stream.Collectors;
import static org.apache.metron.common.Constants.GUID;
@@ -83,10 +84,8 @@ public class ElasticsearchMetaAlertDao implements
MetaAlertDao {
public static final String THREAT_TRIAGE_FIELD =
MetaAlertConstants.THREAT_FIELD_DEFAULT
.replace('.', ':');
public static final String METAALERTS_INDEX = "metaalert_index";
-
public static final
[26/50] [abbrv] metron git commit: METRON-1613 Metaalerts status update broken in Alerts UI (merrimanr) closes apache/metron#1059
METRON-1613 Metaalerts status update broken in Alerts UI (merrimanr) closes
apache/metron#1059
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/10271c41
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/10271c41
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/10271c41
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 10271c41b1df2815041317a36f7b65def5dab831
Parents: 0e48427
Author: merrimanr
Authored: Fri Jun 15 08:17:37 2018 -0500
Committer: merrimanr
Committed: Fri Jun 15 08:17:37 2018 -0500
--
.../src/app/alerts/alert-details/alert-details.component.ts | 5 ++---
.../src/app/alerts/alerts-list/alerts-list.component.ts | 5 ++---
.../alerts/alerts-list/table-view/table-view.component.html | 2 +-
.../alerts/alerts-list/table-view/table-view.component.ts| 8 +++-
.../metron-alerts/src/app/service/update.service.ts | 4
metron-interface/metron-alerts/src/app/utils/constants.ts| 1 -
6 files changed, 8 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/10271c41/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
index 2625502..ffd7dd0 100644
---
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
+++
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
@@ -29,7 +29,7 @@ import {Patch} from '../../model/patch';
import {AlertComment} from './alert-comment';
import {AuthenticationService} from '../../service/authentication.service';
import {MetronDialogBox} from '../../shared/metron-dialog-box';
-import {META_ALERTS_INDEX, META_ALERTS_SENSOR_TYPE} from
'../../utils/constants';
+import {META_ALERTS_SENSOR_TYPE} from '../../utils/constants';
import {CommentAddRemoveRequest} from "../../model/comment-add-remove-request";
export enum AlertState {
@@ -127,7 +127,7 @@ export class AlertDetailsComponent implements OnInit {
this.alertId = params['guid'];
this.alertSourceType = params['source.type.field'];
this.alertIndex = params['index'];
- this.isMetaAlert = (this.alertIndex === META_ALERTS_INDEX &&
this.alertSourceType !== META_ALERTS_SENSOR_TYPE) ? true : false;
+ this.isMetaAlert = this.alertSourceType === META_ALERTS_SENSOR_TYPE;
this.getData();
});
};
@@ -195,7 +195,6 @@ export class AlertDetailsComponent implements OnInit {
let patchRequest = new PatchRequest();
patchRequest.guid = this.alertId;
patchRequest.sensorType = 'metaalert';
- patchRequest.index = META_ALERTS_INDEX;
patchRequest.patch = [new Patch('add', '/name', this.alertName)];
this.updateService.patch(patchRequest).subscribe(rep => {
http://git-wip-us.apache.org/repos/asf/metron/blob/10271c41/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
b/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
index a70f2b4..98459db 100644
---
a/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
+++
b/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
@@ -39,7 +39,7 @@ import {Filter} from '../../model/filter';
import {THREAT_SCORE_FIELD_NAME, TIMESTAMP_FIELD_NAME, ALL_TIME} from
'../../utils/constants';
import {TableViewComponent} from './table-view/table-view.component';
import {Pagination} from '../../model/pagination';
-import {META_ALERTS_SENSOR_TYPE, META_ALERTS_INDEX} from
'../../utils/constants';
+import {META_ALERTS_SENSOR_TYPE} from '../../utils/constants';
import {MetaAlertService} from '../../service/meta-alert.service';
import {Facets} from '../../model/facets';
import { GlobalConfigService } from '../../service/global-config.service';
@@ -396,8 +396,7 @@ export class AlertsListComponent implements OnInit,
OnDestroy {
this.selectedAlerts = [];
this.selectedAlerts = [alert];
this.saveRefreshState();
-let sourceType = (alert.index === META_ALERTS_INDEX &&
!alert.source[this.globalConfig['source.type.field']])
-? META_ALERTS_SENSOR_TYPE :
alert.source[this.globalConfig['source.type.field']];
+let sourceType = alert.source[this.globalConfig['source.type.field']];
let url = '/alerts-list(dialog:details/' + sourceType + '/' +
alert.source.guid
[28/50] [abbrv] metron git commit: METRON-1625 Merge master into Solr feature branch (merrimanr) closes apache/metron#1067
METRON-1625 Merge master into Solr feature branch (merrimanr) closes
apache/metron#1067
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/2bf66503
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/2bf66503
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/2bf66503
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 2bf6650327359ded34d1f96540f13051b8e5c471
Parents: a89a72c
Author: merrimanr
Authored: Wed Jun 20 10:03:44 2018 -0500
Committer: merrimanr
Committed: Wed Jun 20 10:03:44 2018 -0500
--
.../elasticsearch/dao/ElasticsearchDao.java | 4 +
.../dao/ElasticsearchMetaAlertUpdateDao.java| 32 ++--
.../dao/ElasticsearchMetaAlertDaoTest.java | 50 +-
.../indexing/dao/metaalert/MetaScoresTest.java | 26 +++
.../metron/management/KafkaFunctions.java | 173 +--
.../KafkaFunctionsIntegrationTest.java | 165 +-
6 files changed, 418 insertions(+), 32 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/2bf66503/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
--
diff --git
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
index 3eb86ce..59f25f0 100644
---
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
+++
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
@@ -89,6 +89,10 @@ public class ElasticsearchDao implements IndexDao {
return accessConfig;
}
+ public void setAccessConfig(AccessConfig accessConfig) {
+this.accessConfig = accessConfig;
+ }
+
@Override
public synchronized void init(AccessConfig config) {
if (this.client == null) {
http://git-wip-us.apache.org/repos/asf/metron/blob/2bf66503/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java
--
diff --git
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java
index d757dfe..bb79b7a 100644
---
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java
+++
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java
@@ -18,6 +18,7 @@
package org.apache.metron.elasticsearch.dao;
+import static
org.apache.metron.elasticsearch.dao.ElasticsearchMetaAlertDao.METAALERTS_INDEX;
import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
import static org.elasticsearch.index.query.QueryBuilders.nestedQuery;
import static org.elasticsearch.index.query.QueryBuilders.termQuery;
@@ -48,11 +49,14 @@ import
org.apache.metron.indexing.dao.search.InvalidCreateException;
import org.apache.metron.indexing.dao.search.SearchResponse;
import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
+import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.query.InnerHitBuilder;
import org.elasticsearch.index.query.QueryBuilder;
public class ElasticsearchMetaAlertUpdateDao extends
AbstractLuceneMetaAlertUpdateDao {
+ private static final String INDEX_NOT_FOUND_INDICES_KEY = "es.index";
+
private ElasticsearchDao elasticsearchDao;
private MetaAlertRetrieveLatestDao retrieveLatestDao;
private int pageSize;
@@ -169,17 +173,23 @@ public class ElasticsearchMetaAlertUpdateDao extends
AbstractLuceneMetaAlertUpda
} else {
Map> updates = new HashMap<>();
updates.put(update, index);
- // We need to update an alert itself. Only that portion of the update
can be delegated.
- // We still need to get meta alerts potentially associated with it and
update.
- Collection metaAlerts =
getMetaAlertsForAlert(update.getGuid()).getResults()
- .stream()
- .map(searchResult -> new Document(searchResult.getSource(),
searchResult.getId(),
- MetaAlertConstants.METAALERT_TYPE, 0L))
- .collect(Collectors.toList());
- // Each meta alert needs to be updated with the new alert
- for (Document metaAlert : metaAlerts) {
-if (replaceAlertInMetaAlert(metaAlert,
[14/50] [abbrv] metron git commit: METRON-1589 '/api/v1/search/search' fails when 'Solr Zookeeper Urls' has comma separated multiple zookeeper urls (justinleet) closes apache/metron#1040
METRON-1589 '/api/v1/search/search' fails when 'Solr Zookeeper Urls' has comma
separated multiple zookeeper urls (justinleet) closes apache/metron#1040
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9348c608
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9348c608
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9348c608
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 9348c608175afd311177b82fe31737f2f984c175
Parents: 7a071f6
Author: justinleet
Authored: Tue Jun 5 07:44:20 2018 -0400
Committer: leet
Committed: Tue Jun 5 07:44:20 2018 -0400
--
.../org/apache/metron/solr/SolrConstants.java | 1 +
.../metron/solr/dao/SolrColumnMetadataDao.java | 14 +++---
.../org/apache/metron/solr/dao/SolrDao.java | 35 ++
.../metron/solr/dao/SolrMetaAlertDao.java | 2 +-
.../dao/SolrMetaAlertRetrieveLatestDao.java | 2 +-
.../metron/solr/dao/SolrMetaAlertUpdateDao.java | 2 +-
.../apache/metron/solr/writer/SolrWriter.java | 43 +-
.../metron/solr/dao/SolrColumnMetadataTest.java | 11 ++---
.../org/apache/metron/solr/dao/SolrDaoTest.java | 48 ++--
.../metron/solr/dao/SolrMetaAlertDaoTest.java | 4 +-
.../metron/solr/dao/SolrUpdateDaoTest.java | 3 +-
.../SolrIndexingIntegrationTest.java| 4 +-
.../SolrMetaAlertIntegrationTest.java | 5 +-
.../integration/SolrSearchIntegrationTest.java | 4 +-
.../integration/SolrUpdateIntegrationTest.java | 3 +-
.../schema/SchemaValidationIntegrationTest.java | 3 +-
16 files changed, 126 insertions(+), 58 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/9348c608/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/SolrConstants.java
--
diff --git
a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/SolrConstants.java
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/SolrConstants.java
index 879b983..56f1413 100644
---
a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/SolrConstants.java
+++
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/SolrConstants.java
@@ -27,4 +27,5 @@ public class SolrConstants {
public static final String REQUEST_COLLECTIONS_PATH = "/admin/collections";
public static final String RESPONSE_COLLECTIONS = "collections";
public static final String SOLR_WRITER_NAME = "solr";
+ public static final String SOLR_ZOOKEEPER = "solr.zookeeper";
}
http://git-wip-us.apache.org/repos/asf/metron/blob/9348c608/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java
--
diff --git
a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java
index 1e074a9..22c6efa 100644
---
a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java
+++
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java
@@ -29,6 +29,7 @@ import java.util.Map.Entry;
import java.util.Set;
import org.apache.metron.indexing.dao.ColumnMetadataDao;
import org.apache.metron.indexing.dao.search.FieldType;
+import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
@@ -61,10 +62,10 @@ public class SolrColumnMetadataDao implements
ColumnMetadataDao {
solrTypeMap = Collections.unmodifiableMap(fieldTypeMap);
}
- private String zkHost;
+ private transient SolrClient client;
- public SolrColumnMetadataDao(String zkHost) {
-this.zkHost = zkHost;
+ public SolrColumnMetadataDao(SolrClient client) {
+this.client = client;
}
@Override
@@ -115,14 +116,11 @@ public class SolrColumnMetadataDao implements
ColumnMetadataDao {
protected List> getIndexFields(String index)
throws IOException, SolrServerException {
-CloudSolrClient client = new
CloudSolrClient.Builder().withZkHost(zkHost).build();
-client.setDefaultCollection(index);
-
List> indexFields = new ArrayList<>();
// Get all the fields in use, including dynamic fields
LukeRequest lukeRequest = new LukeRequest();
-LukeResponse lukeResponse = lukeRequest.process(client);
+LukeResponse lukeResponse = lukeRequest.process(client, index);
for (Entry field :
lukeResponse.getFieldInfo().entrySet()) {
Map fieldData = new HashMap<>();
fieldData.put("name", field.getValue().getName());
[16/50] [abbrv] metron git commit: METRON-1547 Solr Comment Fields (justinleet) closes apache/metron#1037
METRON-1547 Solr Comment Fields (justinleet) closes apache/metron#1037
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a68d031b
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a68d031b
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a68d031b
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: a68d031b01fe677d84abb0d25aca4f2ceaf90c53
Parents: 9348c60
Author: justinleet
Authored: Tue Jun 5 14:59:29 2018 -0400
Committer: leet
Committed: Tue Jun 5 14:59:29 2018 -0400
--
.../alert-details/alert-details.component.ts| 25 ++-
.../src/app/model/comment-add-remove-request.ts | 25 +++
.../src/app/service/update.service.ts | 27 +++
.../rest/controller/UpdateController.java | 23 +++
.../metron/rest/service/UpdateService.java | 3 +
.../rest/service/impl/UpdateServiceImpl.java| 19 +++
.../UpdateControllerIntegrationTest.java| 92 --
.../elasticsearch/dao/ElasticsearchDao.java | 21 +++
.../dao/ElasticsearchMetaAlertDao.java | 22 +++
.../dao/ElasticsearchMetaAlertUpdateDao.java| 23 +++
.../dao/ElasticsearchUpdateDao.java | 70
.../dao/ElasticsearchMetaAlertDaoTest.java | 17 ++
.../ElasticsearchUpdateIntegrationTest.java | 3 +-
.../apache/metron/indexing/dao/HBaseDao.java| 93 +-
.../apache/metron/indexing/dao/IndexDao.java| 2 +
.../metron/indexing/dao/MultiIndexDao.java | 47 ++
.../indexing/dao/search/AlertComment.java | 130 ++
.../dao/update/CommentAddRemoveRequest.java | 78 +
.../metron/indexing/dao/update/Document.java| 14 +-
.../metron/indexing/dao/update/PatchUtil.java | 50 --
.../metron/indexing/dao/update/UpdateDao.java | 33 +++-
.../apache/metron/indexing/dao/InMemoryDao.java | 17 ++
.../indexing/dao/InMemoryMetaAlertDao.java | 17 ++
.../indexing/dao/UpdateIntegrationTest.java | 169 ++-
.../AbstractLuceneMetaAlertUpdateDaoTest.java | 17 ++
.../integration/HBaseDaoIntegrationTest.java| 79 -
.../src/main/config/schema/bro/schema.xml | 3 +
.../src/main/config/schema/snort/schema.xml | 3 +
.../src/main/config/schema/yaf/schema.xml | 3 +
.../org/apache/metron/solr/dao/SolrDao.java | 41 -
.../metron/solr/dao/SolrMetaAlertDao.java | 21 +++
.../metron/solr/dao/SolrMetaAlertUpdateDao.java | 23 +++
.../metron/solr/dao/SolrRetrieveLatestDao.java | 1 +
.../apache/metron/solr/dao/SolrSearchDao.java | 17 ++
.../apache/metron/solr/dao/SolrUpdateDao.java | 114 -
.../apache/metron/solr/dao/SolrUtilities.java | 36 +++-
.../org/apache/metron/solr/dao/SolrDaoTest.java | 5 +-
.../metron/solr/dao/SolrMetaAlertDaoTest.java | 18 +-
.../metron/solr/dao/SolrUpdateDaoTest.java | 94 +--
.../integration/SolrSearchIntegrationTest.java | 6 +-
.../integration/SolrUpdateIntegrationTest.java | 24 ++-
.../resources/config/test/conf/managed-schema | 3 +
42 files changed, 1400 insertions(+), 128 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
index e1c1685..6a07e08 100644
---
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
+++
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
@@ -30,6 +30,7 @@ import {AlertComment} from './alert-comment';
import {AuthenticationService} from '../../service/authentication.service';
import {MetronDialogBox} from '../../shared/metron-dialog-box';
import {META_ALERTS_INDEX, META_ALERTS_SENSOR_TYPE} from
'../../utils/constants';
+import {CommentAddRemoveRequest} from "../../model/comment-add-remove-request";
export enum AlertState {
NEW, OPEN, ESCALATE, DISMISS, RESOLVE
@@ -204,10 +205,15 @@ export class AlertDetailsComponent implements OnInit {
}
onAddComment() {
-let alertComment = new AlertComment(this.alertCommentStr,
this.authenticationService.getCurrentUserName(), new Date().getTime());
-let tAlertComments = this.alertCommentsWrapper.map(alertsWrapper =>
alertsWrapper.alertComment);
-tAlertComments.unshift(alertComment);
-this.patchAlert(new Patch('add', '/comments', tAlertComments));
+let commentRequest = new CommentAddRemoveRequest();
+commentRequest.guid = this.alertSource.guid;
+commentRequest.comment = this.alertCommentStr;
[13/50] [abbrv] metron git commit: METRON-1577 Solr searches don t include the index of the result (merrimanr) closes apache/metron#1031
METRON-1577 Solr searches dont include the index of the result
(merrimanr) closes apache/metron#1031
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/7a071f6d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/7a071f6d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/7a071f6d
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 7a071f6d64794267f40a0ef1810bd7deaf044eeb
Parents: 49f851e
Author: merrimanr
Authored: Fri May 25 08:41:44 2018 -0500
Committer: merrimanr
Committed: Fri May 25 08:41:44 2018 -0500
--
.../src/app/service/alerts.service.ts | 2 +-
.../apache/metron/rest/config/IndexConfig.java | 3 +-
.../metron-rest/src/main/scripts/metron-rest.sh | 3 +
.../ElasticsearchSearchIntegrationTest.java | 9 +++
.../metron/indexing/util/IndexingCacheUtil.java | 14 -
.../indexing/dao/SearchIntegrationTest.java | 4 ++
.../indexing/util/IndexingCacheUtilTest.java| 66
.../org/apache/metron/solr/SolrConstants.java | 1 +
.../metron/solr/dao/SolrMetaAlertSearchDao.java | 3 +-
.../apache/metron/solr/dao/SolrSearchDao.java | 7 ++-
.../apache/metron/solr/dao/SolrUpdateDao.java | 23 +--
.../apache/metron/solr/dao/SolrUtilities.java | 5 +-
.../apache/metron/solr/writer/SolrWriter.java | 4 +-
.../metron/solr/dao/SolrSearchDaoTest.java | 8 ++-
.../metron/solr/dao/SolrUpdateDaoTest.java | 34 --
.../integration/SolrSearchIntegrationTest.java | 6 ++
16 files changed, 172 insertions(+), 20 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-interface/metron-alerts/src/app/service/alerts.service.ts
--
diff --git a/metron-interface/metron-alerts/src/app/service/alerts.service.ts
b/metron-interface/metron-alerts/src/app/service/alerts.service.ts
index 7324a72..56939d8 100644
--- a/metron-interface/metron-alerts/src/app/service/alerts.service.ts
+++ b/metron-interface/metron-alerts/src/app/service/alerts.service.ts
@@ -30,7 +30,7 @@ export class AlertsService {
}
public escalate(alerts: Alert[]): Observable {
-return this.http.post('/api/v1/alert/escalate', alerts, new
RequestOptions({headers: new Headers(this.defaultHeaders)}))
+return this.http.post('/api/v1/alerts/ui/escalate', alerts, new
RequestOptions({headers: new Headers(this.defaultHeaders)}))
.catch(HttpUtil.handleError);
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java
--
diff --git
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java
index c432c6c..53b10f9 100644
---
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java
+++
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java
@@ -18,6 +18,7 @@
package org.apache.metron.rest.config;
import static org.apache.metron.rest.MetronRestConstants.INDEX_DAO_IMPL;
+import static org.apache.metron.rest.MetronRestConstants.INDEX_WRITER_NAME;
import java.util.Optional;
import org.apache.metron.common.zookeeper.ConfigurationsCache;
@@ -73,7 +74,7 @@ public class IndexConfig {
throw new IllegalStateException("Unable to retrieve the global
config.", e);
}
});
- config.setIndexSupplier(IndexingCacheUtil.getIndexLookupFunction(cache));
+ config.setIndexSupplier(IndexingCacheUtil.getIndexLookupFunction(cache,
environment.getProperty(INDEX_WRITER_NAME)));
config.setTableProvider(TableProvider.create(hbaseProviderImpl, () ->
new HTableProvider()));
config.setKerberosEnabled(environment.getProperty(MetronRestConstants.KERBEROS_ENABLED_SPRING_PROPERTY,
Boolean.class, false));
if (indexDaoImpl == null) {
http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
--
diff --git a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
index 20b535e..21e8128 100644
--- a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
+++ b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
@@ -113,10 +113,13 @@ echo "METRON_REST_CLASSPATH=${METRON_REST_CLASSPATH}"
if [[ ${METRON_RA_INDEXING_WRITER} == "Solr" ]]; then
METRON_INDEX_DAO="
[21/50] [abbrv] metron git commit: Merge branch 'master' into feature/METRON-1416-upgrade-solr
Merge branch 'master' into feature/METRON-1416-upgrade-solr
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0717cfc2
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0717cfc2
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0717cfc2
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 0717cfc25acc80b80121b424bed93db7014a1376
Parents: 4170887
Author: cstella
Authored: Mon Jun 11 21:41:06 2018 -0400
Committer: cstella
Committed: Mon Jun 11 21:41:06 2018 -0400
--
.../org/apache/metron/common/Constants.java | 1 +
.../elasticsearch/dao/ElasticsearchDao.java | 4 ++
.../dao/ElasticsearchMetaAlertDao.java | 30 +-
.../dao/ElasticsearchMetaAlertUpdateDao.java| 4 +-
.../indexing/dao/metaalert/MetaAlertConfig.java | 43
.../AbstractLuceneMetaAlertUpdateDao.java | 3 ++
.../dao/metaalert/MetaAlertIntegrationTest.java | 2 +-
.../AbstractLuceneMetaAlertUpdateDaoTest.java | 26 +---
.../metron/solr/dao/SolrMetaAlertDao.java | 29 +
.../metron/solr/dao/SolrMetaAlertSearchDao.java | 15 ---
.../metron/solr/dao/SolrMetaAlertUpdateDao.java | 2 +-
.../SolrMetaAlertIntegrationTest.java | 28 +
12 files changed, 128 insertions(+), 59 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/0717cfc2/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
--
diff --git
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
index f74660c..4a8bea2 100644
---
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
+++
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
@@ -28,6 +28,7 @@ public class Constants {
public static final long DEFAULT_CONFIGURED_BOLT_TIMEOUT = 5000;
public static final String SENSOR_TYPE = "source.type";
public static final String SENSOR_TYPE_FIELD_PROPERTY = "source.type.field";
+ public static final String THREAT_SCORE_FIELD_PROPERTY =
"threat.triage.score.field";
public static final String ENRICHMENT_TOPIC = "enrichments";
public static final String INDEXING_TOPIC = "indexing";
public static final String ERROR_STREAM = "error";
http://git-wip-us.apache.org/repos/asf/metron/blob/0717cfc2/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
--
diff --git
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
index eae0a39..3eb86ce 100644
---
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
+++
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
@@ -85,6 +85,10 @@ public class ElasticsearchDao implements IndexDao {
//uninitialized.
}
+ public AccessConfig getAccessConfig() {
+return accessConfig;
+ }
+
@Override
public synchronized void init(AccessConfig config) {
if (this.client == null) {
http://git-wip-us.apache.org/repos/asf/metron/blob/0717cfc2/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
--
diff --git
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
index f73a640..649077e 100644
---
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
+++
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
@@ -68,6 +68,7 @@ import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
+import java.util.function.Supplier;
import java.util.stream.Collectors;
import static org.apache.metron.common.Constants.GUID;
@@ -83,10 +84,8 @@ public class ElasticsearchMetaAlertDao implements
MetaAlertDao {
public static final String THREAT_TRIAGE_FIELD =
MetaAlertConstants.THREAT_FIELD_DEFAULT
.replace('.', ':');
public static final String METAALERTS_INDEX = "metaalert_index";
-
public static final
[34/50] [abbrv] metron git commit: Merge branch 'master' into feature/METRON-1416-upgrade-solr
Merge branch 'master' into feature/METRON-1416-upgrade-solr
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e0fc4758
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e0fc4758
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e0fc4758
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: e0fc47582e65e9daf1a0c85dc1b1657166ff14f5
Parents: 1767727 622d50a
Author: justinjleet
Authored: Tue Jun 26 15:34:09 2018 -0400
Committer: justinjleet
Committed: Tue Jun 26 16:07:29 2018 -0400
--
.../configuration/metron-enrichment-env.xml | 12 +
.../configuration/metron-profiler-env.xml | 12 +
.../CURRENT/package/scripts/metron_service.py | 20 +
.../package/scripts/params/params_linux.py |4 +
.../METRON/CURRENT/themes/metron_theme.json | 41 +-
metron-interface/metron-alerts/README.md| 19 +-
.../e2e/alert-details/alert-details.po.ts | 67 +-
.../alert-details-status.e2e-spec.ts| 168 +-
.../alert-filters/alert-filters.e2e-spec.ts | 78 +-
.../alert-filters/alert-filters.po.ts | 31 +-
.../alert-status/alerts-list-status.e2e-spec.ts | 125 +-
.../e2e/alerts-list/alerts-list.e2e-spec.ts | 369 +-
.../e2e/alerts-list/alerts-list.po.ts | 308 +-
.../configure-table/configure-table.e2e-spec.ts | 62 +-
.../meta-alerts/meta-alert.e2e-spec.ts | 248 +-
.../alerts-list/meta-alerts/meta-alert.po.ts| 21 +-
.../save-search/save-search.e2e-spec.ts | 93 +-
.../alerts-list/tree-view/tree-view.e2e-spec.ts | 208 +-
.../e2e/alerts-list/tree-view/tree-view.po.ts | 170 +-
.../metron-alerts/e2e/login/login.e2e-spec.ts | 36 +-
.../metron-alerts/e2e/login/login.po.ts | 44 +-
.../e2e/utils/clean_metron_update_table.ts | 52 +
.../metron-alerts/e2e/utils/e2e_util.ts | 223 +-
.../metron-alerts/package-lock.json | 6163 +++---
metron-interface/metron-alerts/package.json |8 +-
.../metron-alerts/protractor.conf.js| 31 +-
.../alert-details/alert-details.component.ts| 49 +-
.../alert-filters/alert-filters.component.ts|2 +-
.../alerts-list/alerts-list.component.html |8 +-
.../alerts-list/alerts-list.component.scss |3 +-
.../alerts/alerts-list/alerts-list.component.ts |4 +-
.../tree-view/tree-view.component.html |4 +-
.../configure-rows/configure-rows.component.ts |2 +-
.../saved-searches.component.html |4 +-
.../app/shared/group-by/group-by.component.html |2 +-
.../shared/time-range/time-range.component.html | 12 +-
.../shared/time-range/time-range.component.scss |4 +
.../metron-alerts/src/app/utils/constants.ts|1 +
.../src/environments/environment.e2e.ts |3 +-
.../src/environments/environment.js |6 +-
.../src/environments/environment.prod.ts|3 +-
.../src/environments/environment.ts |3 +-
metron-interface/metron-alerts/src/index.html |2 +-
metron-interface/metron-alerts/src/styles.scss | 10 +
44 files changed, 6761 insertions(+), 1974 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/e0fc4758/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
--
diff --cc
metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
index 58d581f,e3e48ab..9d15e93
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
@@@ -148,15 -119,29 +148,35 @@@ def build_global_config_patch(params, p
"op": "add",
"path": "/threat.triage.score.field",
"value": "{{threat_triage_score_field}}"
+ },
+ {
+ "op": "add",
+ "path": "/enrichment.writer.batchSize",
+ "value": "{{enrichment_kafka_writer_batch_size}}"
+ },
+ {
+ "op": "add",
+ "path": "/enrichment.writer.batchTimeout",
+ "value": "{{enrichment_kafka_writer_batch_timeout}}"
+ },
+ {
+ "op": "add",
+ "path": "/profiler.writer.batchSize",
+ "value": "{{profiler_kafka_writer_batch_size}}"
+ },
+ {
+ "op": "add",
+ "path": "/profiler.writer.batchTimeout",
+ "value": "{{profiler_kafka_writer_batch_timeout}}"
}
- ]
"""
+ patch_template = ambari_format(
+ """
+ [
+
[25/50] [abbrv] metron git commit: Merge branch 'master' into feature/METRON-1416-upgrade-solr
Merge branch 'master' into feature/METRON-1416-upgrade-solr Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0e484279 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0e484279 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0e484279 Branch: refs/heads/feature/METRON-1554-pcap-query-panel Commit: 0e4842799f8a3afa15074cb121bdbd205e50fb58 Parents: 4c4452e 828ab71 Author: cstella Authored: Thu Jun 14 12:18:43 2018 -0400 Committer: cstella Committed: Thu Jun 14 12:18:43 2018 -0400 -- NOTICE | 6 + dependencies_with_url.csv | 4 + metron-platform/metron-api/pom.xml | 4 - .../parsers/topology/ParserTopologyBuilder.java | 2 +- .../metron-storm-kafka-override/pom.xml | 83 - .../KafkaSpoutRetryExponentialBackoff.java | 336 +++ .../storm/kafka/spout/internal/Timer.java | 9 +- metron-platform/metron-storm-kafka/pom.xml | 5 - pom.xml | 4 +- site/documentation/index.md | 14 +- 10 files changed, 442 insertions(+), 25 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/0e484279/dependencies_with_url.csv -- http://git-wip-us.apache.org/repos/asf/metron/blob/0e484279/pom.xml --
[24/50] [abbrv] metron git commit: Merge branch 'master' into feature/METRON-1416-upgrade-solr
Merge branch 'master' into feature/METRON-1416-upgrade-solr
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/4c4452e5
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/4c4452e5
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/4c4452e5
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 4c4452e59c93306a049e6879f7b6fed2ee86c74e
Parents: 4768561
Author: cstella
Authored: Tue Jun 12 21:40:01 2018 -0400
Committer: cstella
Committed: Tue Jun 12 21:40:01 2018 -0400
--
.../dao/ElasticsearchSearchDao.java | 30 +--
.../ElasticsearchMetaAlertIntegrationTest.java | 223 ++-
2 files changed, 134 insertions(+), 119 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/4c4452e5/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java
--
diff --git
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java
index 5725534..5cd0a4d 100644
---
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java
+++
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java
@@ -443,22 +443,24 @@ public class ElasticsearchSearchDao implements SearchDao {
private List getGroupResults(GroupRequest groupRequest, int
index, Aggregations aggregations, Map commonColumnMetadata) {
List groups = groupRequest.getGroups();
String field = groups.get(index).getField();
-Terms terms = aggregations.get(getGroupByAggregationName(field));
List searchResultGroups = new ArrayList<>();
-for(Bucket bucket: terms.getBuckets()) {
- GroupResult groupResult = new GroupResult();
- groupResult.setKey(formatKey(bucket.getKey(),
commonColumnMetadata.get(field)));
- groupResult.setTotal(bucket.getDocCount());
- Optional scoreField = groupRequest.getScoreField();
- if (scoreField.isPresent()) {
-Sum score =
bucket.getAggregations().get(getSumAggregationName(scoreField.get()));
-groupResult.setScore(score.getValue());
- }
- if (index < groups.size() - 1) {
-groupResult.setGroupedBy(groups.get(index + 1).getField());
-groupResult.setGroupResults(getGroupResults(groupRequest, index + 1,
bucket.getAggregations(), commonColumnMetadata));
+if(aggregations != null) {
+ Terms terms = aggregations.get(getGroupByAggregationName(field));
+ for (Bucket bucket : terms.getBuckets()) {
+GroupResult groupResult = new GroupResult();
+groupResult.setKey(formatKey(bucket.getKey(),
commonColumnMetadata.get(field)));
+groupResult.setTotal(bucket.getDocCount());
+Optional scoreField = groupRequest.getScoreField();
+if (scoreField.isPresent()) {
+ Sum score =
bucket.getAggregations().get(getSumAggregationName(scoreField.get()));
+ groupResult.setScore(score.getValue());
+}
+if (index < groups.size() - 1) {
+ groupResult.setGroupedBy(groups.get(index + 1).getField());
+ groupResult.setGroupResults(getGroupResults(groupRequest, index + 1,
bucket.getAggregations(), commonColumnMetadata));
+}
+searchResultGroups.add(groupResult);
}
- searchResultGroups.add(groupResult);
}
return searchResultGroups;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/4c4452e5/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
--
diff --git
a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
index 99f0490..224ec56 100644
---
a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
+++
b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
@@ -1,20 +1,21 @@
+
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under
[19/50] [abbrv] metron git commit: METRON-1587 Make collection utility work for HDP search (merrimanr) closes apache/metron#1043
METRON-1587 Make collection utility work for HDP search (merrimanr) closes
apache/metron#1043
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f241f87f
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f241f87f
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f241f87f
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: f241f87f87523f2c6f2203b3f316cff0fd3f20b4
Parents: aed4ffc
Author: merrimanr
Authored: Mon Jun 11 10:09:16 2018 -0500
Committer: merrimanr
Committed: Mon Jun 11 10:09:16 2018 -0500
--
.../package/scripts/indexing_commands.py| 63 +---
.../CURRENT/package/scripts/indexing_master.py | 31 +-
.../package/scripts/params/params_linux.py | 5 ++
metron-platform/metron-solr/README.md | 33 +-
.../src/main/scripts/create_collection.sh | 21 +--
.../src/main/scripts/delete_collection.sh | 18 --
6 files changed, 120 insertions(+), 51 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/f241f87f/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
index eeb2127..69e980b 100755
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
@@ -25,6 +25,7 @@ from resource_management.core.exceptions import Fail
from resource_management.core.logger import Logger
from resource_management.core.resources.system import Execute, File
from resource_management.libraries.functions import format as ambari_format
+from resource_management.libraries.functions.format import format
import metron_service
import metron_security
@@ -85,14 +86,13 @@ class IndexingCommands:
:return: Dict where key is the name of a collection and the
value is a path to file containing the schema definition.
"""
-from params import params
-return {
-"bro": params.bro_schema_path,
-"yaf": params.yaf_schema_path,
-"snort": params.snort_schema_path,
-"error": params.error_schema_path,
-"metaalert": params.meta_schema_path
-}
+return [
+"bro",
+"yaf",
+"snort",
+"error",
+"metaalert"
+]
def is_configured(self):
return self.__configured
@@ -200,6 +200,53 @@ class IndexingCommands:
user=self.__params.metron_user,
err_msg=err_msg.format(template_name))
+def solr_schema_install(self, env):
+from params import params
+env.set_params(params)
+Logger.info("Installing Solr schemas")
+if self.__params.security_enabled:
+metron_security.kinit(self.__params.kinit_path_local,
+ self.__params.solr_keytab_path,
+ self.__params.solr_principal_name,
+ self.__params.solr_user)
+
+commands = IndexingCommands(params)
+for collection_name in commands.get_solr_schemas():
+
+# install the schema
+cmd = format((
+"export ZOOKEEPER={solr_zookeeper_url};"
+"export SECURITY_ENABLED={security_enabled};"
+))
+cmd += "{0}/bin/create_collection.sh {1};"
+
+Execute(
+cmd.format(params.metron_home, collection_name),
+user=self.__params.solr_user)
+
+def solr_schema_delete(self, env):
+from params import params
+env.set_params(params)
+Logger.info("Deleting Solr schemas")
+if self.__params.security_enabled:
+metron_security.kinit(self.__params.kinit_path_local,
+ self.__params.solr_keytab_path,
+ self.__params.solr_principal_name,
+ self.__params.solr_user)
+
+commands = IndexingCommands(params)
+for collection_name in commands.get_solr_schemas():
+# delete the schema
+cmd = format((
+"export ZOOKEEPER={solr_zookeeper_url};"
+"export
[18/50] [abbrv] metron git commit: METRON-1585 SolrRetrieveLatestDao does not use the collection lookup (justinleet via merrimanr) closes apache/metron#1050
METRON-1585 SolrRetrieveLatestDao does not use the collection lookup
(justinleet via merrimanr) closes apache/metron#1050
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/aed4ffcb
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/aed4ffcb
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/aed4ffcb
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: aed4ffcb24cf9a95c1347cccd5dd79fce7c77a6b
Parents: 95e6528
Author: justinleet
Authored: Fri Jun 8 12:16:59 2018 -0500
Committer: merrimanr
Committed: Fri Jun 8 12:16:59 2018 -0500
--
.../org/apache/metron/solr/dao/SolrDao.java | 10 +-
.../metron/solr/dao/SolrRetrieveLatestDao.java | 37 +++-
.../apache/metron/solr/dao/SolrSearchDao.java | 17 --
.../apache/metron/solr/dao/SolrUpdateDao.java | 18 +-
.../apache/metron/solr/dao/SolrUtilities.java | 18 ++
.../org/apache/metron/solr/dao/SolrDaoTest.java | 4 +-
.../metron/solr/dao/SolrSearchDaoTest.java | 2 +-
.../metron/solr/dao/SolrUpdateDaoTest.java | 4 +-
.../SolrRetrieveLatestIntegrationTest.java | 207 +++
.../integration/SolrUpdateIntegrationTest.java | 1 +
10 files changed, 266 insertions(+), 52 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/aed4ffcb/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
--
diff --git
a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
index 7db0ab5..a840bb4 100644
---
a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
+++
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
@@ -22,16 +22,13 @@ import static
org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER;
import com.google.common.base.Splitter;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.function.Function;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.ColumnMetadataDao;
import org.apache.metron.indexing.dao.IndexDao;
import org.apache.metron.indexing.dao.RetrieveLatestDao;
-import org.apache.metron.indexing.dao.search.AlertComment;
import org.apache.metron.indexing.dao.search.FieldType;
import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.search.GroupRequest;
@@ -43,7 +40,6 @@ import
org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
-import org.apache.metron.solr.SolrConstants;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
@@ -92,7 +88,7 @@ public class SolrDao implements IndexDao {
this.accessConfig = config;
this.client = getSolrClient(getZkHosts());
this.solrSearchDao = new SolrSearchDao(this.client, this.accessConfig);
- this.solrRetrieveLatestDao = new SolrRetrieveLatestDao(this.client);
+ this.solrRetrieveLatestDao = new SolrRetrieveLatestDao(this.client,
this.accessConfig);
this.solrUpdateDao = new SolrUpdateDao(this.client,
this.solrRetrieveLatestDao, this.accessConfig);
this.solrColumnMetadataDao = new SolrColumnMetadataDao(this.client);
}
@@ -118,8 +114,8 @@ public class SolrDao implements IndexDao {
}
@Override
- public Document getLatest(String guid, String collection) throws IOException
{
-return this.solrRetrieveLatestDao.getLatest(guid, collection);
+ public Document getLatest(String guid, String sensorType) throws IOException
{
+return this.solrRetrieveLatestDao.getLatest(guid, sensorType);
}
@Override
http://git-wip-us.apache.org/repos/asf/metron/blob/aed4ffcb/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java
--
diff --git
a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java
index b3bc564..40262ba 100644
---
a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java
+++
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java
@@ -19,13 +19,16
[39/50] [abbrv] metron git commit: METRON-1647 Fix logging level score closes apache/incubator-metron#1089
METRON-1647 Fix logging level score closes apache/incubator-metron#1089
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/55d46b0e
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/55d46b0e
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/55d46b0e
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 55d46b0ec630fc926a6deeafaa079ae3841a3677
Parents: 4519f72
Author: Pravin Rahangdale
Authored: Fri Jun 29 09:22:43 2018 -0400
Committer: cstella
Committed: Fri Jun 29 09:22:43 2018 -0400
--
.../main/java/org/apache/metron/maas/service/runner/Runner.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/55d46b0e/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java
--
diff --git
a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java
b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java
index 8f0b9e5..4103d3d 100644
---
a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java
+++
b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java
@@ -177,7 +177,7 @@ public class Runner {
String hostname = RunnerOptions.HOSTNAME.get(cli);
CuratorFramework client = null;
-LOG.error("Running script " + script);
+LOG.info("Running script " + script);
LOG.info("Local Directory Contents");
for(File f : new File(".").listFiles()) {
LOG.info(" " + f.getName());
[48/50] [abbrv] metron git commit: METRON-1644: Support parser chaining closes apache/incubator-metron#1084
http://git-wip-us.apache.org/repos/asf/metron/blob/cbdaee17/use-cases/parser_chaining/README.md
--
diff --git a/use-cases/parser_chaining/README.md
b/use-cases/parser_chaining/README.md
new file mode 100644
index 000..26fd333
--- /dev/null
+++ b/use-cases/parser_chaining/README.md
@@ -0,0 +1,235 @@
+
+# Problem Statement
+
+Aggregating many different types sensors into a single data source (e.g.
+syslog) and ingesting that aggregate sensor into Metron is a common pattern.
It
+is not obvious precisely how to manage these types of aggregate sensors
+as they require two-pass parsing. This document will walk through an
+example of supporting this kind of multi-pass ingest.
+
+Multi-pass parser involves the following requirements:
+* The enveloping parser (e.g. the aggregation format such as syslog or
+ plain CSV) may contain metadata which should be ingested along with the data.
+* The enveloping sensor contains many different sensor types
+
+# High Level Solution
+
+
+
+At a high level, we continue to maintain the architectural invariant of
+a 1-1 relationship between logical sensors and storm topologies.
+Eventually this relationship may become more complex, but at the moment
+the approach is to construct a routing parser which will have two
+responsibilities:
+* Parse the envelope (e.g. syslog data) and extract any metadata fields
+ from the envelope to pass along
+* Route the unfolded data to the appropriate kafka topic associated with
+ the enveloped sensor data
+
+Because the data emitted from the routing parser is just like any data
+emitted from any other parser, in that it is a JSON blob like any
+data emitted from any parser, we will need to adjust the downstream
+parsers to extract the enveloped data from the JSON blob and treat it as
+the data to parse.
+
+# Example
+
+## Preliminaries
+
+We assume that the following environment variables are set:
+* `METRON_HOME` - the home directory for metron
+* `ZOOKEEPER` - The zookeeper quorum (comma separated with port specified:
e.g. `node1:2181` for full-dev)
+* `BROKERLIST` - The Kafka broker list (comma separated with port specified:
e.g. `node1:6667` for full-dev)
+* `ES_HOST` - The elasticsearch master (and port) e.g. `node1:9200` for
full-dev.
+
+Before editing configurations, be sure to pull the configs from zookeeper
locally via
+```
+$METRON_HOME/bin/zk_load_configs.sh --mode PULL -z $ZOOKEEPER -o
$METRON_HOME/config/zookeeper/ -f
+```
+
+
+## The Scenario
+
+Consider the following situation, we have some logs from a Cisco PIX
+device that we would like to ingest. The format is syslog, but multiple
+scenarios exist in the same log file. Specificaly, let's consider the
+sample logs
+[here](http://www.monitorware.com/en/logsamples/cisco-pix-61(2).php).
+
+The log lines in general have the following components:
+* A timestamp
+* A message type tag
+* The message payload that is dependent upon the tag
+
+Let's consider two types of messages that we'd like to parse:
+* Tag `6-302*` which are connection creation and teardown messages e.g. `Built
UDP connection for faddr 198.207.223.240/53337 gaddr 10.0.0.187/53 laddr
192.168.0.2/53`
+* Tag `5-304*` which are URL access events e.g. `192.168.0.2 Accessed URL
66.102.9.99:/`
+
+A couple things are apparent from this:
+* The formats we care about are easy to represent in grok, but are very
+ different and logically represent very different sensors.
+* The syslog loglines output by this device has many types of events that I do
not care
+ about (yet).
+
+We will proceed to create 3 separate parsers:
+* A `pix_syslog_router` parser which will:
+ * Parse the timestamp field
+ * Parse the payload into a field called `data`
+ * Parse the tag into a field called `pix_type`
+ * Route the enveloped messages to the appropriate kafka topic
+based on the tag
+* A `cisco-6-302` and `cisco-5-304` parser which will append to the existing
fields from
+ the `pix_syslog_router` the sensor specific fields based on the tag type.
+
+## Cisco PIX Grok Patterns
+In order to assist in these parsers, we're going to accumulate some grok
+expressions which will help us deal with these various parsers.
+
+* Open a file `~/cisco_patterns` and place the following in there
+```
+CISCO_ACTION Built|Teardown|Deny|Denied|denied|requested|permitted|denied by
ACL|discarded|est-allowed|Dropping|created|deleted
+CISCO_REASON Duplicate TCP SYN|Failed to locate egress interface|Invalid
transport field|No matching connection|DNS Response|DNS Query|(?:%{WORD}\s*)*
+CISCO_DIRECTION Inbound|inbound|Outbound|outbound
+CISCOFW302020_302021 %{CISCO_ACTION:action}(?:%{CISCO_DIRECTION:direction})?
%{WORD:protocol} connection %{GREEDYDATA:ignore} faddr
%{IP:ip_dst_addr}/%{INT:icmp_seq_num}(?:\(%{DATA:fwuser}\))? gaddr
%{IP:ip_src_xlated}/%{INT:icmp_code_xlated} laddr
[35/50] [abbrv] metron git commit: METRON-1645: Check wether the Solr management pack is installed before configuring the solr principal name. closes apache/incubator-metron#1086
METRON-1645: Check wether the Solr management pack is installed before
configuring the solr principal name. closes apache/incubator-metron#1086
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3a6fb38e
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3a6fb38e
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3a6fb38e
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 3a6fb38e55c1b8b9c1fa9ec74ed233c68a18d076
Parents: e0fc475
Author: MohanDV
Authored: Thu Jun 28 09:33:21 2018 -0400
Committer: cstella
Committed: Thu Jun 28 09:33:21 2018 -0400
--
.../METRON/CURRENT/package/scripts/params/params_linux.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/3a6fb38e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index 07132a0..0ab9bcf 100755
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -258,8 +258,9 @@ if security_enabled:
kafka_keytab_path = config['configurations']['kafka-env']['kafka_keytab']
nimbus_seeds = config['configurations']['storm-site']['nimbus.seeds']
-
-solr_principal_name = solr_principal_name.replace('_HOST',
hostname_lowercase)
+# Check wether Solr mpack is installed
+if 'solr-config-env' in config['configurations']:
+solr_principal_name = solr_principal_name.replace('_HOST',
hostname_lowercase)
# Management UI
metron_rest_host = default("/clusterHostInfo/metron_rest_hosts", [hostname])[0]
[09/50] [abbrv] metron git commit: METRON-1421 Create a SolrMetaAlertDao (justinleet) closes apache/metron#970
http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
--
diff --git
a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
index 56406f4..7fca764 100644
---
a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
+++
b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
@@ -39,8 +39,6 @@ import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.integration.InMemoryComponent;
import org.junit.AfterClass;
import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -472,25 +470,15 @@ public abstract class SearchIntegrationTest {
@Multiline
public static String differentTypeFilterQuery;
- protected static IndexDao dao;
protected static InMemoryComponent indexComponent;
- @Before
- public synchronized void setup() throws Exception {
-if(dao == null && indexComponent == null) {
- indexComponent = startIndex();
- loadTestData();
- dao = createDao();
-}
- }
-
@Rule
public ExpectedException thrown = ExpectedException.none();
@Test
public void all_query_returns_all_results() throws Exception {
SearchRequest request = JSONUtils.INSTANCE.load(allQuery,
SearchRequest.class);
-SearchResponse response = dao.search(request);
+SearchResponse response = getIndexDao().search(request);
Assert.assertEquals(10, response.getTotal());
List results = response.getResults();
Assert.assertEquals(10, results.size());
@@ -507,7 +495,7 @@ public abstract class SearchIntegrationTest {
@Test
public void find_one_guid() throws Exception {
GetRequest request = JSONUtils.INSTANCE.load(findOneGuidQuery,
GetRequest.class);
-Optional> response = dao.getLatestResult(request);
+Optional> response =
getIndexDao().getLatestResult(request);
Assert.assertTrue(response.isPresent());
Map doc = response.get();
Assert.assertEquals("bro", doc.get(getSourceTypeField()));
@@ -519,7 +507,7 @@ public abstract class SearchIntegrationTest {
List request = JSONUtils.INSTANCE.load(getAllLatestQuery, new
JSONUtils.ReferenceSupplier>(){});
Map docs = new HashMap<>();
-for(Document doc : dao.getAllLatest(request)) {
+for(Document doc : getIndexDao().getAllLatest(request)) {
docs.put(doc.getGuid(), doc);
}
Assert.assertEquals(2, docs.size());
@@ -532,7 +520,7 @@ public abstract class SearchIntegrationTest {
@Test
public void filter_query_filters_results() throws Exception {
SearchRequest request = JSONUtils.INSTANCE.load(filterQuery,
SearchRequest.class);
-SearchResponse response = dao.search(request);
+SearchResponse response = getIndexDao().search(request);
Assert.assertEquals(3, response.getTotal());
List results = response.getResults();
Assert.assertEquals("snort",
results.get(0).getSource().get(getSourceTypeField()));
@@ -546,7 +534,7 @@ public abstract class SearchIntegrationTest {
@Test
public void sort_query_sorts_results_ascending() throws Exception {
SearchRequest request = JSONUtils.INSTANCE.load(sortQuery,
SearchRequest.class);
-SearchResponse response = dao.search(request);
+SearchResponse response = getIndexDao().search(request);
Assert.assertEquals(10, response.getTotal());
List results = response.getResults();
for (int i = 8001; i < 8011; ++i) {
@@ -557,7 +545,7 @@ public abstract class SearchIntegrationTest {
@Test
public void sort_ascending_with_missing_fields() throws Exception {
SearchRequest request =
JSONUtils.INSTANCE.load(sortAscendingWithMissingFields, SearchRequest.class);
-SearchResponse response = dao.search(request);
+SearchResponse response = getIndexDao().search(request);
Assert.assertEquals(10, response.getTotal());
List results = response.getResults();
Assert.assertEquals(10, results.size());
@@ -575,7 +563,7 @@ public abstract class SearchIntegrationTest {
@Test
public void sort_descending_with_missing_fields() throws Exception {
SearchRequest request =
JSONUtils.INSTANCE.load(sortDescendingWithMissingFields, SearchRequest.class);
-SearchResponse response = dao.search(request);
+SearchResponse response = getIndexDao().search(request);
Assert.assertEquals(10, response.getTotal());
List results = response.getResults();
Assert.assertEquals(10, results.size());
@@ -593,7 +581,7 @@ public abstract class SearchIntegrationTest {
@Test
public void results_are_paginated()
[49/50] [abbrv] metron git commit: METRON-1644: Support parser chaining closes apache/incubator-metron#1084
METRON-1644: Support parser chaining closes apache/incubator-metron#1084
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/cbdaee17
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/cbdaee17
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/cbdaee17
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: cbdaee174dc81280efb0f2aaa8e0028f0c930281
Parents: b2a22b8
Author: cstella
Authored: Tue Jul 10 16:24:32 2018 -0400
Committer: cstella
Committed: Tue Jul 10 16:24:32 2018 -0400
--
.../org/apache/metron/common/Constants.java | 1 -
.../configuration/SensorParserConfig.java | 69 -
.../metadata/DefaultRawMessageStrategy.java | 86 ++
.../metadata/EnvelopedRawMessageStrategy.java | 146 ++
.../common/message/metadata/MetadataUtil.java | 124
.../common/message/metadata/RawMessage.java | 85 ++
.../message/metadata/RawMessageStrategies.java | 81 ++
.../message/metadata/RawMessageStrategy.java| 77 +
.../common/message/metadata/RawMessageUtil.java | 59
.../common/bolt/ConfiguredParserBoltTest.java | 31 +-
.../message/metadata/RawMessageUtilTest.java| 282 +++
.../components/FluxTopologyComponent.java | 9 +
.../metron-parsers/ParserChaining.md| 179
metron-platform/metron-parsers/README.md| 37 ++-
.../apache/metron/parsers/bolt/ParserBolt.java | 60 ++--
.../EnvelopedParserIntegrationTest.java | 209 ++
.../parsers/integration/ParserDriver.java | 2 +-
.../src/test/resources/patterns/cisco_patterns | 6 +
pom.xml | 1 +
use-cases/parser_chaining/README.md | 235
.../message_routing_high_level.svg | 14 +
.../message_routing_high_level.xml | 14 +
22 files changed, 1741 insertions(+), 66 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/cbdaee17/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
--
diff --git
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
index 4a8bea2..5054508 100644
---
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
+++
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
@@ -22,7 +22,6 @@ import java.util.Map;
public class Constants {
- public static final String METADATA_PREFIX = "metron.metadata.";
public static final String ZOOKEEPER_ROOT = "/metron";
public static final String ZOOKEEPER_TOPOLOGY_ROOT = ZOOKEEPER_ROOT +
"/topology";
public static final long DEFAULT_CONFIGURED_BOLT_TIMEOUT = 5000;
http://git-wip-us.apache.org/repos/asf/metron/blob/cbdaee17/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
--
diff --git
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
index 1dfb045..01630c1 100644
---
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
+++
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
@@ -21,6 +21,8 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.metron.common.message.metadata.RawMessageStrategy;
+import org.apache.metron.common.message.metadata.RawMessageStrategies;
import org.apache.metron.common.utils.JSONUtils;
import java.io.IOException;
@@ -29,6 +31,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
/**
* The configuration object that defines a parser for a given sensor. Each
@@ -86,18 +89,25 @@ public class SensorParserConfig implements Serializable {
* transformations. If true, the parser field transformations can access
* parser metadata values.
*
- * By default, this is false and parser metadata is not available
- * to the field transformations.
+ * The default is dependent upon the raw message strategy used:
+ *
+ * The default strategy sets this to false and metadata is not read by
default.
+ * The
[07/50] [abbrv] metron git commit: METRON-1421 Create a SolrMetaAlertDao (justinleet) closes apache/metron#970
http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
--
diff --git
a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
index f1b7102..53ac083 100644
---
a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
+++
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
@@ -18,10 +18,17 @@
package org.apache.metron.solr.dao;
import com.fasterxml.jackson.core.JsonProcessingException;
-import org.apache.metron.common.Constants;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import org.apache.commons.lang3.StringUtils;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.indexing.dao.AccessConfig;
-import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.search.Group;
import org.apache.metron.indexing.dao.search.GroupOrder;
import org.apache.metron.indexing.dao.search.GroupOrderType;
@@ -35,7 +42,6 @@ import org.apache.metron.indexing.dao.search.SearchResponse;
import org.apache.metron.indexing.dao.search.SearchResult;
import org.apache.metron.indexing.dao.search.SortField;
import org.apache.metron.indexing.dao.search.SortOrder;
-import org.apache.metron.indexing.dao.update.Document;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrQuery.ORDER;
@@ -45,25 +51,11 @@ import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.FacetField.Count;
import org.apache.solr.client.solrj.response.PivotField;
import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.stream.Collectors;
-
-import static org.apache.metron.common.Constants.SENSOR_TYPE;
-
public class SolrSearchDao implements SearchDao {
private static final Logger LOG =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -78,6 +70,13 @@ public class SolrSearchDao implements SearchDao {
@Override
public SearchResponse search(SearchRequest searchRequest) throws
InvalidSearchException {
+return search(searchRequest, null);
+ }
+
+ // Allow for the fieldList to be explicitly specified, letting things like
metaalerts expand on them.
+ // If null, use whatever the searchRequest defines.
+ public SearchResponse search(SearchRequest searchRequest, String fieldList)
+ throws InvalidSearchException {
if (searchRequest.getQuery() == null) {
throw new InvalidSearchException("Search query is invalid: null");
}
@@ -89,7 +88,7 @@ public class SolrSearchDao implements SearchDao {
"Search result size must be less than " +
accessConfig.getMaxSearchResults());
}
try {
- SolrQuery query = buildSearchRequest(searchRequest);
+ SolrQuery query = buildSearchRequest(searchRequest, fieldList);
QueryResponse response = client.query(query);
return buildSearchResponse(searchRequest, response);
} catch (SolrException | IOException | SolrServerException e) {
@@ -108,6 +107,7 @@ public class SolrSearchDao implements SearchDao {
.setStart(0)
.setRows(0)
.setQuery(groupRequest.getQuery());
+
query.set("collection", getCollections(groupRequest.getIndices()));
Optional scoreField = groupRequest.getScoreField();
if (scoreField.isPresent()) {
@@ -125,39 +125,10 @@ public class SolrSearchDao implements SearchDao {
}
}
- @Override
- public Document getLatest(String guid, String collection) throws IOException
{
-try {
- SolrDocument solrDocument = client.getById(collection, guid);
- return toDocument(solrDocument);
-} catch (SolrServerException e) {
- throw new IOException(e);
-}
- }
-
- @Override
- public Iterable getAllLatest(List getRequests) throws
IOException {
-Map> collectionIdMap = new HashMap<>();
-for (GetRequest getRequest: getRequests) {
- Collection ids =
collectionIdMap.getOrDefault(getRequest.getSensorType(), new HashSet<>());
-
[42/50] [abbrv] metron git commit: METRON-1642: KafkaWriter should be able choose the topic from a field in addition to topology construction time closes apache/incubator-metron#1082
METRON-1642: KafkaWriter should be able choose the topic from a field in
addition to topology construction time closes apache/incubator-metron#1082
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/097ce950
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/097ce950
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/097ce950
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 097ce95030e31bf9bd2af74aa56dc03348f7fef7
Parents: fc9ff85
Author: cstella
Authored: Tue Jul 3 09:29:47 2018 -0400
Committer: cstella
Committed: Tue Jul 3 09:29:47 2018 -0400
--
metron-platform/metron-parsers/README.md| 1 +
.../integration/WriterBoltIntegrationTest.java | 75 +++-
metron-platform/metron-writer/README.md | 24 +++
.../apache/metron/writer/kafka/KafkaWriter.java | 43 +--
.../metron/writer/kafka/KafkaWriterTest.java| 63
5 files changed, 199 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/097ce950/metron-platform/metron-parsers/README.md
--
diff --git a/metron-platform/metron-parsers/README.md
b/metron-platform/metron-parsers/README.md
index d79b9ce..7ddfdea 100644
--- a/metron-platform/metron-parsers/README.md
+++ b/metron-platform/metron-parsers/README.md
@@ -166,6 +166,7 @@ then it is assumed to be a regex and will match any topic
matching the pattern (
* `batchTimeout` : The timeout after which a batch will be flushed even if
batchSize has not been met. Optional.
If unspecified, or set to `0`, it defaults to a system-determined duration
which is a fraction of the Storm
parameter `topology.message.timeout.secs`. Ignored if batchSize is `1`,
since this disables batching.
+ * The kafka writer can be configured within the parser config as well.
(This is all configured a priori, but this is convenient for overriding the
settings). See [here](../metron-writer/README.md#kafka-writer)
* `fieldTransformations` : An array of complex objects representing the
transformations to be done on the message generated from the parser before
writing out to the kafka topic.
* `spoutParallelism` : The kafka spout parallelism (default to `1`). This can
be overridden on the command line.
* `spoutNumTasks` : The number of tasks for the spout (default to `1`). This
can be overridden on the command line.
http://git-wip-us.apache.org/repos/asf/metron/blob/097ce950/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
--
diff --git
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
index dfadfdc..99506de 100644
---
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
+++
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertThat;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import java.io.IOException;
import java.io.Serializable;
@@ -32,6 +33,7 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
+import java.util.function.Predicate;
import javax.annotation.Nullable;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.hadoop.hbase.util.Bytes;
@@ -66,7 +68,7 @@ public class WriterBoltIntegrationTest extends
BaseIntegrationTest {
@Override
public boolean isValid(Map input, Map
validationConfig, Map globalConfig, Context context) {
- if (input.get("action").equals("invalid")) {
+ if (input.get("action") != null &&
input.get("action").equals("invalid")) {
return false;
}
return true;
@@ -105,6 +107,69 @@ public class WriterBoltIntegrationTest extends
BaseIntegrationTest {
@Multiline
public static String parserConfigJSON;
+ /**
+ * {
+ *"parserClassName" : "org.apache.metron.parsers.csv.CSVParser",
+ *"sensorTopic": "dummy",
+ *"outputTopic": "output",
+ *"errorTopic": "parser_error",
+ *"parserConfig": {
+ *"batchSize" : 1,
+ *"columns" : {
+ *"name" : 0,
+ *"dummy" : 1
+ *},
+ * "kafka.topicField" : "route_field"
+ *}
+ *,"fieldTransformations" : [
+ *
[27/50] [abbrv] metron git commit: Merge branch 'master' into feature/METRON-1416-upgrade-solr
Merge branch 'master' into feature/METRON-1416-upgrade-solr
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a89a72c3
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a89a72c3
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a89a72c3
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: a89a72c3b2ceb5066fd7a310636507ce274c66ad
Parents: 10271c4 b081e80
Author: cstella
Authored: Fri Jun 15 20:32:18 2018 -0400
Committer: cstella
Committed: Fri Jun 15 20:32:18 2018 -0400
--
.../alert-details/alert-details.component.html | 4 +--
.../alert-details/alert-details.component.ts| 24 ++---
.../alerts-list/alerts-list.component.html | 4 +--
.../alerts/alerts-list/alerts-list.component.ts | 13 ++
.../src/app/alerts/alerts-list/query-builder.ts | 5 ++--
.../table-view/table-view.component.html| 18 ++---
.../table-view/table-view.component.ts | 19 --
.../tree-view/tree-view.component.html | 16 ++--
.../tree-view/tree-view.component.ts| 27 +---
.../meta-alerts/meta-alerts.component.html | 2 +-
.../alerts/meta-alerts/meta-alerts.component.ts | 11 +---
.../src/app/model/group-request.ts | 2 +-
.../src/app/service/global-config.service.ts| 16 +---
.../metron-alerts/src/app/utils/constants.ts| 3 +--
.../ElasticsearchMetaAlertIntegrationTest.java | 5 ++--
15 files changed, 120 insertions(+), 49 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
--
diff --cc
metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
index ffd7dd0,c939f04..39d47ac
---
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
+++
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
@@@ -29,8 -30,8 +30,9 @@@ import {Patch} from '../../model/patch'
import {AlertComment} from './alert-comment';
import {AuthenticationService} from '../../service/authentication.service';
import {MetronDialogBox} from '../../shared/metron-dialog-box';
- import {META_ALERTS_SENSOR_TYPE} from '../../utils/constants';
-import {META_ALERTS_INDEX, META_ALERTS_SENSOR_TYPE} from
'../../utils/constants';
+import {CommentAddRemoveRequest} from
"../../model/comment-add-remove-request";
++import {META_ALERTS_SENSOR_TYPE} from '../../utils/constants';
+ import { GlobalConfigService } from '../../service/global-config.service';
export enum AlertState {
NEW, OPEN, ESCALATE, DISMISS, RESOLVE
http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
--
diff --cc
metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
index 98459db,4496e37..07f7015
---
a/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
+++
b/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
@@@ -36,10 -36,10 +36,10 @@@ import {AlertSearchDirective} from '../
import {SearchResponse} from '../../model/search-response';
import {ElasticsearchUtils} from '../../utils/elasticsearch-utils';
import {Filter} from '../../model/filter';
- import {THREAT_SCORE_FIELD_NAME, TIMESTAMP_FIELD_NAME, ALL_TIME} from
'../../utils/constants';
+ import {TIMESTAMP_FIELD_NAME, ALL_TIME} from '../../utils/constants';
import {TableViewComponent} from './table-view/table-view.component';
import {Pagination} from '../../model/pagination';
-import {META_ALERTS_SENSOR_TYPE, META_ALERTS_INDEX} from
'../../utils/constants';
+import {META_ALERTS_SENSOR_TYPE} from '../../utils/constants';
import {MetaAlertService} from '../../service/meta-alert.service';
import {Facets} from '../../model/facets';
import { GlobalConfigService } from '../../service/global-config.service';
http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-interface/metron-alerts/src/app/alerts/alerts-list/table-view/table-view.component.html
--
http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-interface/metron-alerts/src/app/alerts/alerts-list/table-view/table-view.component.ts
--
http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-interface/metron-alerts/src/app/utils/constants.ts
[37/50] [abbrv] metron git commit: METRON-1619: Stellar empty collections should be considered false in boolean expressions closes apache/incubator-metron#1064
METRON-1619: Stellar empty collections should be considered false in boolean
expressions closes apache/incubator-metron#1064
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/c4c790db
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/c4c790db
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/c4c790db
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: c4c790dbe4c481c8f19293ef9d6134539674d253
Parents: 81282de
Author: cstella
Authored: Thu Jun 28 15:57:08 2018 -0400
Committer: cstella
Committed: Thu Jun 28 15:57:08 2018 -0400
--
metron-stellar/stellar-common/README.md | 14
.../metron/stellar/common/StellarCompiler.java | 82 +---
.../org/apache/metron/stellar/dsl/Token.java| 1 +
.../stellar/dsl/functions/BasicStellarTest.java | 25 ++
.../metron/stellar/dsl/functions/MatchTest.java | 35 +
use-cases/forensic_clustering/README.md | 2 +-
use-cases/geographic_login_outliers/README.md | 5 +-
use-cases/typosquat_detection/README.md | 2 +-
8 files changed, 149 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/c4c790db/metron-stellar/stellar-common/README.md
--
diff --git a/metron-stellar/stellar-common/README.md
b/metron-stellar/stellar-common/README.md
index d8d09b4..0dd8e16 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -54,6 +54,20 @@ The Stellar language supports the following:
* The ability to have parenthesis to make order of operations explicit
* User defined functions, including Lambda expressions
+### Boolean Expressions
+
+Variables may be used in boolean expressions and variables which are not
+explicitly boolean may be interpreted as booleans subject to the
+following rules:
+* Similar to python and javascript, empty collections (e.g. `[]`) will be
+ interpreted as `false`
+* Similar to python and javascript, missing variables will be
+ interpreted as `false`
+* Variables set to `null` will be interpreted as `false`
+
+Otherwise, boolean variables will be interpreted as their values
+reflect.
+
### Stellar Language Keywords
The following keywords need to be single quote escaped in order to be used in
Stellar expressions:
http://git-wip-us.apache.org/repos/asf/metron/blob/c4c790db/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
--
diff --git
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
index 72f0d0a..8a328a2 100644
---
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
+++
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
@@ -29,7 +29,10 @@ import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
+
+import com.google.common.collect.Iterables;
import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.metron.stellar.common.evaluators.ArithmeticEvaluator;
@@ -104,6 +107,52 @@ public class StellarCompiler extends StellarBaseListener {
return tokenDeque;
}
+/**
+ * When treating empty or missing values as false, we need to ensure we
ONLY do so in a conditional context.
+ * @param tokenValueType
+ * @return
+ */
+private boolean isConditionalContext(Class tokenValueType) {
+ return tokenValueType != null && (
+ tokenValueType == BooleanArg.class
+|| tokenValueType == IfExpr.class
+|| tokenValueType == MatchClauseCheckExpr.class
+ );
+}
+
+/**
+ * Determine if a token and value is an empty list in the appropriate
conditional context
+ * @param token
+ * @param value
+ * @return
+ */
+private boolean isEmptyList(Token token, Object value) {
+ if(value != null && isConditionalContext(token.getUnderlyingType())) {
+if (value instanceof Iterable) {
+ return Iterables.isEmpty((Iterable) value);
+} else if (value instanceof Map) {
+ return ((Map) value).isEmpty();
+}
+else {
+ return false;
+}
+ }else {
+return false;
+ }
+}
+
+/**
+ * Determine if a token is missing in a conditional context.
+ * @param token
+ * @return
+ */
+private boolean isBoolean(Token token, Object value)
[38/50] [abbrv] metron git commit: METRON-1621: Sorting alerts table by score closes apache/incubator-metron#1088
METRON-1621: Sorting alerts table by score closes apache/incubator-metron#1088
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/4519f721
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/4519f721
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/4519f721
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 4519f72104b294c2f0eb8135181a3fa68eee9088
Parents: c4c790d
Author: tiborm
Authored: Fri Jun 29 09:17:27 2018 -0400
Committer: cstella
Committed: Fri Jun 29 09:17:27 2018 -0400
--
.../alert-filters/alert-filters.e2e-spec.ts | 2 +-
.../e2e/alerts-list/alerts-list.po.ts | 6 +-
.../meta-alerts/meta-alert.e2e-spec.ts | 2 +-
.../table-view/table-view.e2e-spec.ts | 90
.../e2e/alerts-list/table-view/table-view.po.ts | 26 ++
.../alerts-list/tree-view/tree-view.e2e-spec.ts | 4 +-
.../e2e/mock-data/alerts_ui_e2e_index.data | 16 ++--
.../metron-alerts/e2e/utils/e2e_util.ts | 62 --
.../metron-alerts/protractor.conf.js| 1 +
.../table-view/table-view.component.html| 10 +--
10 files changed, 172 insertions(+), 47 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/4519f721/metron-interface/metron-alerts/e2e/alerts-list/alert-filters/alert-filters.e2e-spec.ts
--
diff --git
a/metron-interface/metron-alerts/e2e/alerts-list/alert-filters/alert-filters.e2e-spec.ts
b/metron-interface/metron-alerts/e2e/alerts-list/alert-filters/alert-filters.e2e-spec.ts
index d9caf7c..6783d3d 100644
---
a/metron-interface/metron-alerts/e2e/alerts-list/alert-filters/alert-filters.e2e-spec.ts
+++
b/metron-interface/metron-alerts/e2e/alerts-list/alert-filters/alert-filters.e2e-spec.ts
@@ -47,7 +47,7 @@ describe('Test spec for facet filters', function() {
});
it('should display facets data', async function() : Promise {
-let facetValues = [ 'enrichm...:country 3', 'ip_dst_addr 8', 'ip_src_addr
2', 'source:type 1' ];
+let facetValues = [ 'enrichm...:country 3', 'ip_dst_addr 8', 'ip_src_addr
6', 'source:type 1' ];
await page.navgateToAlertList();
expect(await page.getFacetsTitle()).toEqualBcoz('Filters', 'for Title as
Filters');
http://git-wip-us.apache.org/repos/asf/metron/blob/4519f721/metron-interface/metron-alerts/e2e/alerts-list/alerts-list.po.ts
--
diff --git a/metron-interface/metron-alerts/e2e/alerts-list/alerts-list.po.ts
b/metron-interface/metron-alerts/e2e/alerts-list/alerts-list.po.ts
index 13aeb27..389d218 100644
--- a/metron-interface/metron-alerts/e2e/alerts-list/alerts-list.po.ts
+++ b/metron-interface/metron-alerts/e2e/alerts-list/alerts-list.po.ts
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-import {browser, element, by, protractor} from 'protractor';
+import {browser, element, by, protractor, ElementArrayFinder} from
'protractor';
import * as moment from 'moment/moment';
import {
waitForElementVisibility, waitForElementPresence, waitForElementInVisibility,
@@ -446,10 +446,6 @@ export class MetronAlertsPage {
.element(by.xpath('../..')).all(by.css('td a')).get(8).getText();
}
- sortTable(colName: string) {
-element.all(by.css('table thead
th')).all(by.linkText(colName)).get(0).click();
- }
-
getCellValue(rowIndex: number, colIndex: number, previousText: string) {
let cellElement = element.all(by.css('table tbody
tr')).get(rowIndex).all(by.css('td')).get(colIndex);
return this.waitForTextChange(cellElement, previousText).then(() =>
cellElement.getText());
http://git-wip-us.apache.org/repos/asf/metron/blob/4519f721/metron-interface/metron-alerts/e2e/alerts-list/meta-alerts/meta-alert.e2e-spec.ts
--
diff --git
a/metron-interface/metron-alerts/e2e/alerts-list/meta-alerts/meta-alert.e2e-spec.ts
b/metron-interface/metron-alerts/e2e/alerts-list/meta-alerts/meta-alert.e2e-spec.ts
index 5425523..5bf7fd8 100644
---
a/metron-interface/metron-alerts/e2e/alerts-list/meta-alerts/meta-alert.e2e-spec.ts
+++
b/metron-interface/metron-alerts/e2e/alerts-list/meta-alerts/meta-alert.e2e-spec.ts
@@ -158,7 +158,7 @@ describe('Test spec for meta alerts workflow', function() {
'source:type': '1',
'ip_dst_addr': '7',
'enrichm...:country': '3',
- 'ip_src_addr': '2'
+ 'ip_src_addr': '4'
};
let alertsInMetaAlerts = [
'82f8046d-d...03b17480dd',
http://git-wip-us.apache.org/repos/asf/metron/blob/4519f721/metron-interface/metron-alerts/e2e/alerts-list/table-view/table-view.e2e-spec.ts
[12/50] [abbrv] metron git commit: METRON-1421 Create a SolrMetaAlertDao (justinleet) closes apache/metron#970
METRON-1421 Create a SolrMetaAlertDao (justinleet) closes apache/metron#970 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/49f851e0 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/49f851e0 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/49f851e0 Branch: refs/heads/feature/METRON-1554-pcap-query-panel Commit: 49f851e0b8c2ffa1cdd7c8f169bed3dfa07cf35c Parents: eb33666 Author: justinleet Authored: Wed May 23 10:32:34 2018 -0400 Committer: leet Committed: Wed May 23 10:32:34 2018 -0400 -- metron-analytics/metron-profiler/.gitignore |1 + .../apache/metron/rest/config/IndexConfig.java | 12 +- .../rest/service/impl/MetaAlertServiceImpl.java |5 +- .../rest/service/impl/SearchServiceImpl.java|2 +- .../rest/service/impl/UpdateServiceImpl.java|2 +- .../MetaAlertControllerIntegrationTest.java |6 +- .../UpdateControllerIntegrationTest.java|4 +- .../elasticsearch/dao/ElasticsearchDao.java | 65 +- .../dao/ElasticsearchMetaAlertDao.java | 641 ++- ...ElasticsearchMetaAlertRetrieveLatestDao.java | 44 + .../dao/ElasticsearchMetaAlertSearchDao.java| 110 ++ .../dao/ElasticsearchMetaAlertUpdateDao.java| 219 .../dao/ElasticsearchRetrieveLatestDao.java | 151 +++ .../dao/ElasticsearchSearchDao.java | 102 -- .../dao/ElasticsearchUpdateDao.java | 10 +- .../elasticsearch/utils/ElasticsearchUtils.java | 64 ++ .../elasticsearch/dao/ElasticsearchDaoTest.java | 70 +- .../dao/ElasticsearchMetaAlertDaoTest.java | 164 +-- .../ElasticsearchMetaAlertIntegrationTest.java | 986 ++--- .../ElasticsearchSearchIntegrationTest.java | 64 +- .../ElasticsearchUpdateIntegrationTest.java | 84 +- .../components/ElasticSearchComponent.java | 26 +- metron-platform/metron-indexing/README.md |2 +- metron-platform/metron-indexing/pom.xml |8 +- .../metron/indexing/dao/AccessConfig.java | 10 + .../apache/metron/indexing/dao/IndexDao.java| 141 +-- .../metron/indexing/dao/MetaAlertDao.java | 154 --- .../metron/indexing/dao/RetrieveLatestDao.java | 67 ++ .../metaalert/DeferredMetaAlertIndexDao.java| 42 + .../metaalert/MetaAlertAddRemoveRequest.java|1 - .../indexing/dao/metaalert/MetaAlertConfig.java | 74 ++ .../dao/metaalert/MetaAlertConstants.java | 30 + .../indexing/dao/metaalert/MetaAlertDao.java| 77 ++ .../metaalert/MetaAlertRetrieveLatestDao.java | 25 + .../dao/metaalert/MetaAlertSearchDao.java | 35 + .../dao/metaalert/MetaAlertUpdateDao.java | 146 +++ .../indexing/dao/metaalert/MetaScores.java | 52 +- .../AbstractLuceneMetaAlertUpdateDao.java | 334 ++ .../metron/indexing/dao/search/SearchDao.java | 22 +- .../indexing/dao/search/SearchResponse.java | 10 +- .../metron/indexing/dao/update/PatchUtil.java | 50 + .../metron/indexing/dao/update/UpdateDao.java | 47 + .../metron/indexing/util/IndexingCacheUtil.java | 35 + .../indexing/dao/InMemoryMetaAlertDao.java | 69 +- .../indexing/dao/SearchIntegrationTest.java | 60 +- .../indexing/dao/UpdateIntegrationTest.java | 87 +- .../dao/metaalert/MetaAlertIntegrationTest.java | 1012 ++ .../indexing/dao/metaalert/MetaScoresTest.java | 75 ++ .../AbstractLuceneMetaAlertUpdateDaoTest.java | 854 +++ .../integration/IndexingIntegrationTest.java|4 +- metron-platform/metron-pcap-backend/.gitignore |1 + metron-platform/metron-solr/pom.xml |4 +- .../src/main/config/schema/bro/schema.xml |3 + .../src/main/config/schema/metaalert/schema.xml | 39 +- .../src/main/config/schema/snort/schema.xml |3 + .../src/main/config/schema/yaf/schema.xml |3 + .../org/apache/metron/solr/dao/SolrDao.java | 37 +- .../metron/solr/dao/SolrMetaAlertDao.java | 285 +++-- .../dao/SolrMetaAlertRetrieveLatestDao.java | 77 ++ .../metron/solr/dao/SolrMetaAlertSearchDao.java | 211 .../metron/solr/dao/SolrMetaAlertUpdateDao.java | 216 .../metron/solr/dao/SolrRetrieveLatestDao.java | 81 ++ .../apache/metron/solr/dao/SolrSearchDao.java | 127 +-- .../apache/metron/solr/dao/SolrUpdateDao.java | 51 +- .../apache/metron/solr/dao/SolrUtilities.java | 92 ++ .../org/apache/metron/solr/dao/SolrDaoTest.java | 61 +- .../metron/solr/dao/SolrMetaAlertDaoTest.java | 137 +++ .../metron/solr/dao/SolrSearchDaoTest.java | 176 ++- .../metron/solr/dao/SolrUpdateDaoTest.java | 19 +- .../metron/solr/dao/SolrUtilitiesTest.java | 48 + .../SolrIndexingIntegrationTest.java|5 +- .../SolrMetaAlertIntegrationTest.java | 397 +++
[29/50] [abbrv] metron git commit: METRON-1609 Elasticsearch settings in Ambari should not be required if Solr is the indexer (nickwallen) closes apache/metron#1056
METRON-1609 Elasticsearch settings in Ambari should not be required if Solr is
the indexer (nickwallen) closes apache/metron#1056
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ed4dfb93
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ed4dfb93
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ed4dfb93
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: ed4dfb93641c9c9a84b61b5daddbebd962b3b20c
Parents: 2bf6650
Author: nickwallen
Authored: Thu Jun 21 11:28:30 2018 -0400
Committer: nickallen
Committed: Thu Jun 21 11:28:30 2018 -0400
--
.../METRON/CURRENT/configuration/metron-env.xml | 32 --
.../package/scripts/indexing_commands.py| 4 +-
.../CURRENT/package/scripts/indexing_master.py | 10 +-
.../CURRENT/package/scripts/metron_service.py | 101 ++-
4 files changed, 113 insertions(+), 34 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/ed4dfb93/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
index 0b64f3a..cdef7cf 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
@@ -81,30 +81,54 @@
metron
Name of Elasticsearch Cluster
Elasticsearch Cluster Name
+
+true
+
-
+
es_hosts
Comma delimited list of Elasticsearch Master Hosts:
eshost1,eshost2
Elasticsearch Hosts
+
+true
+
es_binary_port
9300
Elasticsearch binary port. (9300)
Elasticsearch Binary Port
+
+true
+
es_http_port
9200
Elasticsearch HTTP port. (9200)
Elasticsearch HTTP port
+
+true
+
+
+
+es_date_format
+Elasticsearch Date Format
+.MM.dd.HH
+Elasticsearch Date Format
+
+true
+
solr_zookeeper_url
{{zookeeper_quorum}}
Comma delimited list of Zookeeper Urls:
zkhost1:2181,zkhost1:2181
Solr Zookeeper Urls
+
+true
+
storm_rest_addr
@@ -137,10 +161,4 @@
true
-
-es_date_format
-Elasticsearch Date Format
-.MM.dd.HH
-Elasticsearch Date Format
-
http://git-wip-us.apache.org/repos/asf/metron/blob/ed4dfb93/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
index 69e980b..4802add 100755
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
@@ -30,7 +30,6 @@ from resource_management.libraries.functions.format import
format
import metron_service
import metron_security
-
# Wrap major operations and functionality in this class
class IndexingCommands:
__params = None
@@ -394,7 +393,6 @@ class IndexingCommands:
is_random_access_running =
topologies[self.__random_access_indexing_topology] in ['ACTIVE', 'REBALANCING']
return is_random_access_running
-
def is_topology_active(self, env):
return self.is_batch_topology_active(env) and
self.is_random_access_topology_active(env)
@@ -403,6 +401,8 @@ class IndexingCommands:
Performs a service check for Indexing.
:param env: Environment
"""
+metron_service.check_indexer_parameters()
+
Logger.info('Checking Kafka topics for Indexing')
metron_service.check_kafka_topics(self.__params, self.__get_topics())
[50/50] [abbrv] metron git commit: Resolve merge conflicts with master.
Resolve merge conflicts with master. Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f30e9cc1 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f30e9cc1 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f30e9cc1 Branch: refs/heads/feature/METRON-1554-pcap-query-panel Commit: f30e9cc1fb9b6985e7ad7582c764d035b5cb0e1e Parents: 9cee51e cbdaee1 Author: Michael Miklavcic Authored: Tue Jul 10 19:31:56 2018 -0600 Committer: Michael Miklavcic Committed: Tue Jul 10 19:31:56 2018 -0600 -- dependencies_with_url.csv |3 + .../metron/maas/service/runner/Runner.java |2 +- metron-analytics/metron-profiler/.gitignore |1 + .../profiler/bolt/ProfileBuilderBolt.java |4 +- metron-deployment/Kerberos-manual-setup.md | 441 ++--- .../ansible/playbooks/metron_full_install.yml |4 + .../ansible/playbooks/solr_install.yml | 40 + .../roles/ambari_config/vars/single_node_vm.yml |1 + .../roles/sensor-stubs/templates/start-bro-stub | 25 +- .../sensor-stubs/templates/start-snort-stub | 25 +- .../roles/sensor-stubs/templates/start-yaf-stub | 26 +- .../ansible/roles/solr/defaults/main.yml| 23 + .../ansible/roles/solr/meta/main.yml| 19 + .../ansible/roles/solr/tasks/main.yml | 36 + metron-deployment/packaging/ambari/.gitignore |1 + .../packaging/ambari/metron-mpack/pom.xml |8 +- .../METRON/CURRENT/configuration/metron-env.xml | 38 +- .../configuration/metron-indexing-env.xml | 32 +- .../common-services/METRON/CURRENT/metainfo.xml |4 + .../package/scripts/indexing_commands.py| 76 +- .../CURRENT/package/scripts/indexing_master.py | 48 +- .../CURRENT/package/scripts/metron_service.py | 102 +- .../package/scripts/params/params_linux.py | 21 +- .../package/scripts/params/status_params.py |3 + .../CURRENT/package/scripts/rest_commands.py|1 + .../METRON/CURRENT/themes/metron_theme.json | 18 +- .../docker/rpm-docker/SPECS/metron.spec | 17 + metron-interface/metron-alerts/.gitignore |1 + metron-interface/metron-alerts/README.md|3 +- .../alert-filters/alert-filters.e2e-spec.ts |2 +- .../e2e/alerts-list/alerts-list.po.ts |6 +- .../meta-alerts/meta-alert.e2e-spec.ts |2 +- .../table-view/table-view.e2e-spec.ts | 90 + .../e2e/alerts-list/table-view/table-view.po.ts | 26 + .../alerts-list/tree-view/tree-view.e2e-spec.ts |4 +- .../e2e/mock-data/alerts_ui_e2e_index.data | 16 +- .../metron-alerts/e2e/utils/e2e_util.ts | 62 +- metron-interface/metron-alerts/karma.conf.js|2 +- .../metron-alerts/protractor.conf.js|1 + .../alert-details/alert-details.component.ts| 135 +- .../alert-filters.component.spec.ts | 14 +- .../alerts-list/alerts-list.component.html |5 +- .../alerts/alerts-list/alerts-list.component.ts |7 +- .../table-view/table-view.component.html| 12 +- .../table-view/table-view.component.spec.ts | 49 +- .../table-view/table-view.component.ts | 16 +- .../tree-view/tree-view.component.spec.ts | 32 +- .../tree-view/tree-view.component.ts| 14 +- .../configure-rows.component.spec.ts| 18 +- .../save-search/save-search.component.spec.ts | 16 + .../saved-searches.component.spec.ts| 27 +- .../metron-alerts/src/app/app.component.spec.ts | 26 +- .../src/app/login/login.component.spec.ts | 40 +- .../src/app/model/comment-add-remove-request.ts | 25 + .../src/app/service/update.service.ts | 31 +- .../shared/collapse/collapse.component.spec.ts | 12 +- .../app/shared/collapse/collapse.component.ts |2 +- .../date-picker/date-picker.component.spec.ts |4 + .../directives/alert-search.directive.spec.ts | 21 - .../alert-severity-hexagon.directive.spec.ts|5 +- .../directives/nav-content.directive.spec.ts|6 +- .../list-group/list-group.component.spec.ts |2 +- .../metron-sorter.component.spec.ts | 14 +- .../metron-table-pagination.component.spec.ts |2 +- .../shared/pipes/center-ellipses.pipe.spec.ts |2 +- .../pipes/column-name-translate.pipe.spec.ts|2 +- .../src/app/shared/pipes/map-keys.pipe.spec.ts |2 +- .../app/shared/switch/switch.component.spec.ts |2 +- .../time-range/time-range.component.spec.ts | 12 +- .../metron-alerts/src/app/utils/constants.ts|1 - metron-interface/metron-rest/README.md |1 + .../apache/metron/rest/config/IndexConfig.java | 14 +- .../rest/controller/UpdateController.java | 23 + .../metron/rest/service/UpdateService.java |3 +
[02/50] [abbrv] metron git commit: METRON-1540 Solr Integration tests should use actual schemas (justinleet) closes apache/metron#1005
http://git-wip-us.apache.org/repos/asf/metron/blob/832296f0/metron-platform/metron-solr/src/test/resources/config/bro/conf/solrconfig.xml
--
diff --git
a/metron-platform/metron-solr/src/test/resources/config/bro/conf/solrconfig.xml
b/metron-platform/metron-solr/src/test/resources/config/bro/conf/solrconfig.xml
deleted file mode 100644
index fff9d84..000
---
a/metron-platform/metron-solr/src/test/resources/config/bro/conf/solrconfig.xml
+++ /dev/null
@@ -1,1601 +0,0 @@
-
-
-
-
-
-
-
-
- 7.2.0
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- ${solr.data.dir:}
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-${solr.lock.type:native}
-
-
-
-
-
-
-
-
-
-
-
-
- true
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- ${solr.ulog.dir:}
- ${solr.ulog.numVersionBuckets:65536}
-
-
-
-
- ${solr.autoCommit.maxTime:15000}
- false
-
-
-
-
-
- ${solr.autoSoftCommit.maxTime:-1}
-
-
-
-
-
-
-
-
-
-
-
-
-
-1024
-
-
-
--1
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-true
-
-
-
-
-
- 20
-
-
- 200
-
-
-
-
-
-
-
-
-
-
-
- static firstSearcher warming in solrconfig.xml
-
-
-
-
-
-false
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- explicit
- 10
-
-
-
- false
-
-
-
-
-
-
-
-
-
-
-
-
-
- explicit
- json
- true
- text
-
-
-
-
-
-
- explicit
-
-
- velocity
- browse
- layout
- Solritas
-
-
- edismax
-
- text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
- title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0
-
- 100%
- *:*
- 10
- *,score
-
-
- text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
- title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0
-
- text,features,name,sku,id,manu,cat,title,description,keywords,author,resourcename
- 3
-
-
- on
- true
- cat
- manu_exact
- content_type
- author_s
- ipod
- GB
- 1
- cat,inStock
- after
- price
- 0
- 600
- 50
- popularity
- 0
- 10
- 3
- manufacturedate_dt
- NOW/YEAR-10YEARS
- NOW
- +1YEAR
- before
- after
-
-
- on
- content features title name
- true
- html
- b
- /b
- 0
- title
- 0
- name
- 3
- 200
- content
- 750
-
-
- on
- false
- 5
- 2
- 5
- true
- true
- 5
- 3
-
-
-
-
- spellcheck
-
-
-
-
-
-
- text
-
-
-
-
-
-
-
-
- true
- ignored_
-
-
- true
- links
- ignored_
-
-
-
-
-
-
-
-
-text_general
-
-
-
-
-
- default
- text
- solr.DirectSolrSpellChecker
-
- internal
-
- 0.5
-
- 2
-
- 1
-
- 5
-
- 4
-
- 0.01
-
-
-
-
-
- wordbreak
- solr.WordBreakSolrSpellChecker
- name
- true
- true
- 10
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- default
- wordbreak
- on
- true
- 10
- 5
- 5
- true
- true
- 10
- 5
-
-
- spellcheck
-
-
-
-
-
-
- mySuggester
- FuzzyLookupFactory
- DocumentDictionaryFactory
- cat
- price
- string
- false
-
-
-
-
-
- true
- 10
-
-
- suggest
-
-
-
-
-
-
-
-
-
-
- true
-
-
- tvComponent
-
-
-
-
-
-
-
-
- lingo3g
- true
- com.carrotsearch.lingo3g.Lingo3GClusteringAlgorithm
- clustering/carrot2
-
-
-
- lingo
- org.carrot2.clustering.lingo.LingoClusteringAlgorithm
- clustering/carrot2
-
-
-
- stc
- org.carrot2.clustering.stc.STCClusteringAlgorithm
- clustering/carrot2
-
-
-
- kmeans
- org.carrot2.clustering.kmeans.BisectingKMeansClusteringAlgorithm
- clustering/carrot2
-
-
-
-
-
-
- true
- true
-
- name
-
- id
-
- features
-
- true
-
-
-
-
[11/50] [abbrv] metron git commit: METRON-1421 Create a SolrMetaAlertDao (justinleet) closes apache/metron#970
http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
--
diff --git
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
index 24f7a27..98dc66d 100644
---
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
+++
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
@@ -36,18 +36,24 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.commons.lang.StringUtils;
import org.apache.metron.common.configuration.writer.WriterConfiguration;
import org.apache.metron.common.utils.HDFSUtils;
import org.apache.metron.common.utils.ReflectionUtils;
+import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.search.SearchResult;
import org.apache.metron.netty.utils.NettyRuntimeWrapper;
import org.apache.metron.stellar.common.utils.ConversionUtils;
import org.codehaus.jackson.map.ObjectMapper;
+import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -321,4 +327,62 @@ public class ElasticsearchUtils {
return json;
}
+
+ /**
+ * Elasticsearch queries default to 10 records returned. Some internal
queries require that all
+ * results are returned. Rather than setting an arbitrarily high size, this
method pages through results
+ * and returns them all in a single SearchResponse.
+ * @param qb A QueryBuilder that provides the query to be run.
+ * @return A SearchResponse containing the appropriate results.
+ */
+ public static SearchResponse queryAllResults(TransportClient
transportClient,
+ QueryBuilder qb,
+ String index,
+ int pageSize
+ ) {
+SearchRequestBuilder searchRequestBuilder = transportClient
+.prepareSearch(index)
+.addStoredField("*")
+.setFetchSource(true)
+.setQuery(qb)
+.setSize(pageSize);
+org.elasticsearch.action.search.SearchResponse esResponse =
searchRequestBuilder
+.execute()
+.actionGet();
+List allResults = getSearchResults(esResponse);
+long total = esResponse.getHits().getTotalHits();
+if (total > pageSize) {
+ int pages = (int) (total / pageSize) + 1;
+ for (int i = 1; i < pages; i++) {
+int from = i * pageSize;
+searchRequestBuilder.setFrom(from);
+esResponse = searchRequestBuilder
+.execute()
+.actionGet();
+allResults.addAll(getSearchResults(esResponse));
+ }
+}
+SearchResponse searchResponse = new SearchResponse();
+searchResponse.setTotal(total);
+searchResponse.setResults(allResults);
+return searchResponse;
+ }
+
+ /**
+ * Transforms a list of Elasticsearch SearchHits to a list of SearchResults
+ * @param searchResponse An Elasticsearch SearchHit to be converted.
+ * @return The list of SearchResults for the SearchHit
+ */
+ protected static List getSearchResults(
+ org.elasticsearch.action.search.SearchResponse searchResponse) {
+return Arrays.stream(searchResponse.getHits().getHits()).map(searchHit -> {
+ SearchResult searchResult = new SearchResult();
+ searchResult.setId(searchHit.getId());
+ searchResult.setSource(searchHit.getSource());
+ searchResult.setScore(searchHit.getScore());
+ searchResult.setIndex(searchHit.getIndex());
+ return searchResult;
+}
+).collect(Collectors.toList());
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java
--
diff --git
a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java
b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java
index ca1b860..6c3c327 100644
---
[32/50] [abbrv] metron git commit: Merge remote-tracking branch 'origin/master' into feature/METRON-1416-upgrade-solr
Merge remote-tracking branch 'origin/master' into feature/METRON-1416-upgrade-solr Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1767727a Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1767727a Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1767727a Branch: refs/heads/feature/METRON-1554-pcap-query-panel Commit: 1767727a70c05c9e611fa9b0474e6900ad9594cd Parents: 975923e b575da1 Author: Nick Allen Authored: Fri Jun 22 11:41:31 2018 -0400 Committer: Nick Allen Committed: Fri Jun 22 11:41:31 2018 -0400 -- dev-utilities/committer-utils/prepare-commit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) --
[06/50] [abbrv] metron git commit: METRON-1421 Create a SolrMetaAlertDao (justinleet) closes apache/metron#970
http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
--
diff --git
a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
index 1cf8c34..4bc9f8a 100644
---
a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
+++
b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
@@ -19,9 +19,13 @@ package org.apache.metron.solr.integration.components;
import com.google.common.base.Function;
import java.util.Collection;
+import java.util.Map.Entry;
import java.util.stream.Collectors;
+import org.apache.metron.common.Constants;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants;
import org.apache.metron.integration.InMemoryComponent;
import org.apache.metron.integration.UnableToStartException;
+import org.apache.metron.solr.dao.SolrUtilities;
import org.apache.metron.solr.writer.MetronSolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
@@ -29,6 +33,7 @@ import org.apache.solr.client.solrj.embedded.JettyConfig;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.cloud.MiniSolrCloudCluster;
import org.apache.solr.common.SolrDocument;
@@ -40,13 +45,15 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.solr.common.SolrInputDocument;
+import org.apache.zookeeper.KeeperException;
public class SolrComponent implements InMemoryComponent {
public static class Builder {
+
private int port = 8983;
private String solrXmlPath =
"../metron-solr/src/test/resources/solr/solr.xml";
-private Map collections = new HashMap<>();
+private Map initialCollections = new HashMap<>();
private Function postStartCallback;
public Builder withPort(int port) {
@@ -59,8 +66,8 @@ public class SolrComponent implements InMemoryComponent {
return this;
}
-public Builder addCollection(String name, String configPath) {
- collections.put(name, configPath);
+public Builder addInitialCollection(String name, String configPath) {
+ initialCollections.put(name, configPath);
return this;
}
@@ -69,9 +76,8 @@ public class SolrComponent implements InMemoryComponent {
return this;
}
-public SolrComponent build() throws Exception {
- if (collections.isEmpty()) throw new Exception("Must add at least 1
collection");
- return new SolrComponent(port, solrXmlPath, collections,
postStartCallback);
+public SolrComponent build() {
+ return new SolrComponent(port, solrXmlPath, initialCollections,
postStartCallback);
}
}
@@ -81,7 +87,8 @@ public class SolrComponent implements InMemoryComponent {
private MiniSolrCloudCluster miniSolrCloudCluster;
private Function postStartCallback;
- private SolrComponent(int port, String solrXmlPath, Map
collections, Function postStartCallback) throws Exception {
+ private SolrComponent(int port, String solrXmlPath, Map
collections,
+ Function postStartCallback) {
this.port = port;
this.solrXmlPath = solrXmlPath;
this.collections = collections;
@@ -93,14 +100,17 @@ public class SolrComponent implements InMemoryComponent {
try {
File baseDir = Files.createTempDirectory("solrcomponent").toFile();
baseDir.deleteOnExit();
- miniSolrCloudCluster = new MiniSolrCloudCluster(1, baseDir.toPath(),
JettyConfig.builder().setPort(port).build());
+ miniSolrCloudCluster = new MiniSolrCloudCluster(1, baseDir.toPath(),
+ JettyConfig.builder().setPort(port).build());
for(String name: collections.keySet()) {
String configPath = collections.get(name);
miniSolrCloudCluster.uploadConfigSet(new File(configPath).toPath(),
name);
CollectionAdminRequest.createCollection(name, 1,
1).process(miniSolrCloudCluster.getSolrClient());
}
- if (postStartCallback != null) postStartCallback.apply(this);
-} catch(Exception e) {
+ if (postStartCallback != null) {
+postStartCallback.apply(this);
+ }
+} catch (Exception e) {
throw new UnableToStartException(e.getMessage(), e);
}
}
@@ -108,21 +118,18 @@ public class SolrComponent implements InMemoryComponent {
@Override
public void stop() {
try {
+ miniSolrCloudCluster.deleteAllCollections();
[33/50] [abbrv] metron git commit: METRON-1634 Alerts UI add comment doesn t immediately show up. (merrimanr) closes apache/metron#1077
METRON-1634 Alerts UI add comment doesnt immediately show up. (merrimanr)
closes apache/metron#1077
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/622d50a1
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/622d50a1
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/622d50a1
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 622d50a168ef7056377d3a3143bc47d2d57ad67f
Parents: 436cec4
Author: merrimanr
Authored: Tue Jun 26 10:42:06 2018 -0500
Committer: rmerriman
Committed: Tue Jun 26 10:42:06 2018 -0500
--
.../alert-details/alert-details.component.ts| 32
1 file changed, 19 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/622d50a1/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
index c939f04..efc7e84 100644
---
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
+++
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
@@ -99,7 +99,7 @@ export class AlertDetailsComponent implements OnInit {
this.alertSource = alertSource;
this.selectedAlertState =
this.getAlertState(alertSource['alert_status']);
this.alertSources = (alertSource.metron_alert &&
alertSource.metron_alert.length > 0) ? alertSource.metron_alert : [alertSource];
- this.setComments(alertSource);
+ this.setComments(alertSource['comments'] || []);
if (fireToggleEditor) {
this.toggleNameEditor();
@@ -107,8 +107,7 @@ export class AlertDetailsComponent implements OnInit {
});
}
- setComments(alert) {
-let alertComments = alert['comments'] ? alert['comments'] : [];
+ setComments(alertComments) {
this.alertCommentsWrapper = alertComments.map(alertComment =>
new AlertCommentWrapper(alertComment, moment(new
Date(alertComment.timestamp)).fromNow()));
}
@@ -222,22 +221,25 @@ export class AlertDetailsComponent implements OnInit {
}
onAddComment() {
-let alertComment = new AlertComment(this.alertCommentStr,
this.authenticationService.getCurrentUserName(), new Date().getTime());
-let tAlertComments = this.alertCommentsWrapper.map(alertsWrapper =>
alertsWrapper.alertComment);
-tAlertComments.unshift(alertComment);
-this.patchAlert(new Patch('add', '/comments', tAlertComments));
+let newComment = new AlertComment(this.alertCommentStr,
this.authenticationService.getCurrentUserName(), new Date().getTime());
+let alertComments = this.alertCommentsWrapper.map(alertsWrapper =>
alertsWrapper.alertComment);
+alertComments.unshift(newComment);
+this.setComments(alertComments);
+this.patchAlert(new Patch('add', '/comments', alertComments), () => {
+ let previousComments = this.alertCommentsWrapper.map(alertsWrapper =>
alertsWrapper.alertComment)
+ .filter(alertComment => alertComment !== newComment);
+ this.setComments(previousComments);
+});
}
- patchAlert(patch: Patch) {
+ patchAlert(patch: Patch, onPatchError) {
let patchRequest = new PatchRequest();
patchRequest.guid = this.alertSource.guid;
patchRequest.index = this.alertIndex;
patchRequest.patch = [patch];
patchRequest.sensorType = this.alertSourceType;
-this.updateService.patch(patchRequest).subscribe(() => {
- this.getData();
-});
+this.updateService.patch(patchRequest).subscribe(() => {}, onPatchError);
}
onDeleteComment(index: number) {
@@ -250,8 +252,12 @@ export class AlertDetailsComponent implements OnInit {
this.metronDialogBox.showConfirmationMessage(commentText).subscribe(response =>
{
if (response) {
-this.alertCommentsWrapper.splice(index, 1);
-this.patchAlert(new Patch('add', '/comments',
this.alertCommentsWrapper.map(alertsWrapper => alertsWrapper.alertComment)));
+let deletedCommentWrapper = this.alertCommentsWrapper.splice(index,
1)[0];
+this.patchAlert(new Patch('add', '/comments',
this.alertCommentsWrapper.map(alertsWrapper => alertsWrapper.alertComment)), ()
=> {
+ // add the deleted comment back
+ this.alertCommentsWrapper.unshift(deletedCommentWrapper);
+ this.alertCommentsWrapper.sort((a, b) => b.alertComment.timestamp -
a.alertComment.timestamp);
+});
}
});
}
[43/50] [abbrv] metron git commit: METRON-1635 Alerts UI status update doesn t immediately show up (merrimanr) closes apache/metron#1080
METRON-1635 Alerts UI status update doesnt immediately show up
(merrimanr) closes apache/metron#1080
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/93276f87
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/93276f87
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/93276f87
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 93276f87b7dfb41b6fd1e6aa6d42895e422fe912
Parents: 097ce95
Author: merrimanr
Authored: Tue Jul 3 08:37:41 2018 -0500
Committer: rmerriman
Committed: Tue Jul 3 08:37:41 2018 -0500
--
.../alert-details/alert-details.component.ts| 68 +---
1 file changed, 30 insertions(+), 38 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/93276f87/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
index e68a8e0..2af79a8 100644
---
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
+++
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
@@ -93,20 +93,20 @@ export class AlertDetailsComponent implements OnInit {
return false;
}
- getData(fireToggleEditor = false) {
+ getData() {
this.alertCommentStr = '';
this.searchService.getAlert(this.alertSourceType,
this.alertId).subscribe(alertSource => {
- this.alertSource = alertSource;
- this.selectedAlertState =
this.getAlertState(alertSource['alert_status']);
- this.alertSources = (alertSource.metron_alert &&
alertSource.metron_alert.length > 0) ? alertSource.metron_alert : [alertSource];
+ this.setAlert(alertSource);
this.setComments(alertSource['comments'] || []);
-
- if (fireToggleEditor) {
-this.toggleNameEditor();
- }
});
}
+ setAlert(alertSource) {
+this.alertSource = alertSource;
+this.alertSources = (alertSource.metron_alert &&
alertSource.metron_alert.length > 0) ? alertSource.metron_alert : [alertSource];
+this.selectedAlertState = this.getAlertState(alertSource['alert_status']);
+ }
+
setComments(alertComments) {
this.alertCommentsWrapper = alertComments.map(alertComment =>
new AlertCommentWrapper(alertComment, moment(new
Date(alertComment.timestamp)).fromNow()));
@@ -149,53 +149,39 @@ export class AlertDetailsComponent implements OnInit {
}
processOpen() {
-let tAlert = new Alert();
-tAlert.source = this.alertSource;
-
-this.selectedAlertState = AlertState.OPEN;
-this.updateService.updateAlertState([tAlert], 'OPEN').subscribe(results =>
{
- this.getData();
-});
+this.updateAlertState('OPEN');
}
processNew() {
-let tAlert = new Alert();
-tAlert.source = this.alertSource;
-
-this.selectedAlertState = AlertState.NEW;
-this.updateService.updateAlertState([tAlert], 'NEW').subscribe(results => {
- this.getData();
-});
+this.updateAlertState('NEW');
}
processEscalate() {
+this.updateAlertState('ESCALATE');
+
let tAlert = new Alert();
tAlert.source = this.alertSource;
-
-this.selectedAlertState = AlertState.ESCALATE;
-this.updateService.updateAlertState([tAlert],
'ESCALATE').subscribe(results => {
- this.getData();
-});
this.alertsService.escalate([tAlert]).subscribe();
}
processDismiss() {
-let tAlert = new Alert();
-tAlert.source = this.alertSource;
-
-this.selectedAlertState = AlertState.DISMISS;
-this.updateService.updateAlertState([tAlert], 'DISMISS').subscribe(results
=> {
- this.getData();
-});
+this.updateAlertState('DISMISS');
}
processResolve() {
+this.updateAlertState('RESOLVE');
+ }
+
+ updateAlertState(state: string) {
let tAlert = new Alert();
tAlert.source = this.alertSource;
-this.selectedAlertState = AlertState.RESOLVE;
-this.updateService.updateAlertState([tAlert], 'RESOLVE').subscribe(results
=> {
- this.getData();
+let previousAlertStatus = this.alertSource['alert_status'];
+this.alertSource['alert_status'] = state;
+this.setAlert(this.alertSource);
+this.updateService.updateAlertState([tAlert], state).subscribe(() => {},
() => {
+ this.alertSource['alert_status'] = previousAlertStatus;
+ this.setAlert(this.alertSource);
});
}
@@ -213,8 +199,14 @@ export class AlertDetailsComponent implements OnInit {
patchRequest.sensorType = 'metaalert';
patchRequest.patch = [new Patch('add', '/name', this.alertName)];
+ let previousName =
[30/50] [abbrv] metron git commit: METRON-1629 Update Solr documentation (merrimanr via justinleet) closes apache/metron#1072
METRON-1629 Update Solr documentation (merrimanr via justinleet) closes apache/metron#1072 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/6159c6f9 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/6159c6f9 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/6159c6f9 Branch: refs/heads/feature/METRON-1554-pcap-query-panel Commit: 6159c6f997851fabe1b3c21cc0e06bf49c49d9f7 Parents: ed4dfb9 Author: merrimanr Authored: Fri Jun 22 07:48:27 2018 -0400 Committer: leet Committed: Fri Jun 22 07:48:27 2018 -0400 -- metron-interface/metron-alerts/README.md | 3 ++- metron-interface/metron-rest/README.md| 1 + metron-platform/metron-indexing/README.md | 14 ++ metron-platform/metron-parsers/README.md | 2 ++ metron-platform/metron-solr/README.md | 26 ++ 5 files changed, 45 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/6159c6f9/metron-interface/metron-alerts/README.md -- diff --git a/metron-interface/metron-alerts/README.md b/metron-interface/metron-alerts/README.md index 2d34505..486a885 100644 --- a/metron-interface/metron-alerts/README.md +++ b/metron-interface/metron-alerts/README.md @@ -40,7 +40,8 @@ Sorting has a similar caveat, in that if we are matching on multiple alerts, the Alerts that are contained in a a meta alert are generally excluded from search results, because a user has already grouped them in a meaningful way. ## Prerequisites -* The Metron REST application should be up and running and Elasticsearch should have some alerts populated by Metron topologies +* The Metron REST application should be up and running +* Elasticsearch or Solr should have some alerts populated by Metron topologies, depending on which real-time store is enabled * The Management UI should be installed (which includes [Express](https://expressjs.com/)) * The alerts can be populated using Full Dev or any other setup * UI is developed using angular4 and uses angular-cli http://git-wip-us.apache.org/repos/asf/metron/blob/6159c6f9/metron-interface/metron-rest/README.md -- diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md index 2a6a0e0..44594f7 100644 --- a/metron-interface/metron-rest/README.md +++ b/metron-interface/metron-rest/README.md @@ -22,6 +22,7 @@ This module provides a RESTful API for interacting with Metron. ## Prerequisites * A running Metron cluster +* A running real-time store, either Elasticsearch or Solr depending on which one is enabled * Java 8 installed * Storm CLI and Metron topology scripts (start_parser_topology.sh, start_enrichment_topology.sh, start_elasticsearch_topology.sh) installed * A relational database http://git-wip-us.apache.org/repos/asf/metron/blob/6159c6f9/metron-platform/metron-indexing/README.md -- diff --git a/metron-platform/metron-indexing/README.md b/metron-platform/metron-indexing/README.md index 5a35d62..46e511b 100644 --- a/metron-platform/metron-indexing/README.md +++ b/metron-platform/metron-indexing/README.md @@ -77,6 +77,20 @@ Alerts can be grouped, after appropriate searching, into a set of alerts called ### Elasticsearch Metron comes with built-in templates for the default sensors for Elasticsearch. When adding a new sensor, it will be necessary to add a new template defining the output fields appropriately. In addition, there is a requirement for a field `alert` of type `nested` for Elasticsearch 2.x installs. This is detailed at [Using Metron with Elasticsearch 2.x](../metron-elasticsearch/README.md#using-metron-with-elasticsearch-2x) +### Solr + +Metron comes with built-in schemas for the default sensors for Solr. When adding a new sensor, it will be necessary to add a new schema defining the output fields appropriately. In addition, these fields are used internally by Metron and also required: + +* `` +* `` +* `` +* `` +* `` + +The unique key should be set to `guid` by including `guid` in the schema. + +It is strongly suggested the `fieldTypes` match those in the built-in schemas. + ### Indexing Configuration Examples For a given sensor, the following scenarios would be indicated by the following cases: http://git-wip-us.apache.org/repos/asf/metron/blob/6159c6f9/metron-platform/metron-parsers/README.md -- diff --git a/metron-platform/metron-parsers/README.md b/metron-platform/metron-parsers/README.md index 8254baf..d79b9ce 100644 --- a/metron-platform/metron-parsers/README.md +++
[45/50] [abbrv] metron git commit: METRON-1652 Document X-Pack Common Problem (nickwallen) closes apache/metron#1092
METRON-1652 Document X-Pack Common Problem (nickwallen) closes
apache/metron#1092
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/4eb9a8eb
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/4eb9a8eb
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/4eb9a8eb
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 4eb9a8ebca733bbae96e7e19e9064c5e71666884
Parents: d90f7cf
Author: nickwallen
Authored: Thu Jul 5 11:07:59 2018 -0400
Committer: nickallen
Committed: Thu Jul 5 11:07:59 2018 -0400
--
metron-deployment/Kerberos-manual-setup.md | 441 +---
1 file changed, 243 insertions(+), 198 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/4eb9a8eb/metron-deployment/Kerberos-manual-setup.md
--
diff --git a/metron-deployment/Kerberos-manual-setup.md
b/metron-deployment/Kerberos-manual-setup.md
index ecda858..e3de148 100644
--- a/metron-deployment/Kerberos-manual-setup.md
+++ b/metron-deployment/Kerberos-manual-setup.md
@@ -538,203 +538,248 @@ In order to correct this, you should:
X-Pack
--
-First, stop the random_access_indexing topology through the Storm UI or from
the CLI, e.g.
+1. Stop the random_access_indexing topology through the Storm UI or from the
CLI, e.g.
-```
-storm kill random_access_indexing
-```
-
-Here are instructions for enabling X-Pack with Elasticsearch and Kibana:
https://www.elastic.co/guide/en/x-pack/5.6/installing-xpack.html
-
-You need to be sure to add the appropriate username and password for
Elasticsearch and Kibana to enable external connections from Metron components.
e.g. the following will create a user "transport_client_user" with password
"changeme" and "superuser" credentials.
-
-```
-sudo /usr/share/elasticsearch/bin/x-pack/users useradd transport_client_user
-p changeme -r superuser
-```
-
-Once you've picked a password to connect to ES, you need to upload a 1-line
file to HDFS with that password in it. Metron will use this file to securely
read the password in order to connect to ES securely.
-
-Here is an example using "changeme" as the password
-
-```
-echo changeme > /tmp/xpack-password
-sudo -u hdfs hdfs dfs -mkdir /apps/metron/elasticsearch/
-sudo -u hdfs hdfs dfs -put /tmp/xpack-password /apps/metron/elasticsearch/
-sudo -u hdfs hdfs dfs -chown metron:metron
/apps/metron/elasticsearch/xpack-password
-```
-
-New settings have been added to configure the Elasticsearch client. By default
the client will run as the normal ES prebuilt transport client. If you enable
X-Pack you should set the es.client.class as shown below.
-
-Add the es settings to global.json
-
-```
-/usr/metron/0.5.1/config/zookeeper/global.json ->
-
- "es.client.settings" : {
- "es.client.class" :
"org.elasticsearch.xpack.client.PreBuiltXPackTransportClient",
- "es.xpack.username" : "transport_client_user",
- "es.xpack.password.file" : "/apps/metron/elasticsearch/xpack-password"
- }
-```
-
-Submit the update to Zookeeper
-
-```
-$METRON_HOME/bin/zk_load_configs.sh -m PUSH -i METRON_HOME/config/zookeeper/
-z $ZOOKEEPER
-```
-
-The last step before restarting the topology is to create a custom X-Pack
shaded and relocated jar. This is up to you because of licensing restrictions,
but here is a sample Maven pom file that should help.
-
-```
-
-
-http://maven.apache.org/POM/4.0.0;
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
-4.0.0
-org.elasticsearch
-elasticsearch-xpack-shaded
-elasticsearch-xpack-shaded
-jar
-5.6.2
-
-
-elasticsearch-releases
-https://artifacts.elastic.co/maven
-
-true
-
-
-false
-
-
-
-
-
-org.elasticsearch.client
-x-pack-transport
-5.6.2
-
-
-com.fasterxml.jackson.dataformat
-jackson-dataformat-yaml
-
-
-com.fasterxml.jackson.dataformat
-jackson-dataformat-cbor
-
-
-com.fasterxml.jackson.core
-jackson-core
-
-
-org.slf4j
-slf4j-api
-
-
-org.slf4j
-slf4j-log4j12
-
-
-log4j
-log4j
-
-
-org.apache.logging.log4j
-log4j-api
-
-
-
-
[36/50] [abbrv] metron git commit: METRON-1646 Sensor Stubs should work when kerberized (nickwallen) closes apache/metron#1087
METRON-1646 Sensor Stubs should work when kerberized (nickwallen) closes
apache/metron#1087
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/81282de2
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/81282de2
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/81282de2
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 81282de288d94b1fa8c57917ff6a34079eca77e1
Parents: 3a6fb38
Author: nickwallen
Authored: Thu Jun 28 14:07:32 2018 -0400
Committer: nickallen
Committed: Thu Jun 28 14:07:32 2018 -0400
--
.../roles/sensor-stubs/templates/start-bro-stub | 25 ++-
.../sensor-stubs/templates/start-snort-stub | 25 ++-
.../roles/sensor-stubs/templates/start-yaf-stub | 26 ++--
3 files changed, 57 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/81282de2/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub
--
diff --git
a/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub
b/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub
index 979de3d..24027b3 100644
--- a/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub
+++ b/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub
@@ -21,14 +21,20 @@
# simulates the behavior of a sensor by sending canned telemetry data
# to a Kafka topic.
#
-# a subset of the canned data is randomly selected and is sent in
+# a subset of the canned data is randomly selected and is sent in
# batches. the timestamp of the message is altered to match current
-# system time. the number of messages sent in each batch, along with
+# system time. the number of messages sent in each batch, along with
# the time delay between batches can be configured.
#
# start-bro-stub
#
+METRON_SYSCONFIG="/etc/default/metron"
+if [ -f "$METRON_SYSCONFIG" ]; then
+ set -a
+ . "$METRON_SYSCONFIG"
+fi
+
#
# how long to delay between each 'batch' in seconds.
#
@@ -36,20 +42,27 @@ DELAY=${1:-{{ sensor_stubs_delay }}}
#
# how many messages to send in each 'batch'. the messages are drawn randomly
-# from the entire set of canned data.
+# from the entire set of canned data.
#
COUNT=${2:-{{ sensor_stubs_count }}}
INPUT="{{ sensor_stubs_data }}/bro.out"
PRODUCER="{{ kafka_home }}/bin/kafka-console-producer.sh"
TOPIC="bro"
+SECURITY_ENABLED=${SECURITY_ENABLED:-false}
+KAFKA_SECURITY_PROTOCOL=${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}
+
+if [ ${SECURITY_ENABLED,,} == 'true' ]; then
+ echo "Security enabled"
+ kinit -kt $METRON_SERVICE_KEYTAB $METRON_PRINCIPAL_NAME
+fi
while true; do
-
+
# transform the bro timestamp and push to kafka
SEARCH="\"ts\"\:[0-9]\+\."
REPLACE="\"ts\"\:`date +%s`\."
- shuf -n $COUNT $INPUT | sed -e "s/$SEARCH/$REPLACE/g" | $PRODUCER
--broker-list {{ kafka_broker_url }} --topic $TOPIC
-
+ shuf -n $COUNT $INPUT | sed -e "s/$SEARCH/$REPLACE/g" | $PRODUCER
--broker-list $BROKERLIST --topic $TOPIC --security-protocol
$KAFKA_SECURITY_PROTOCOL
+
sleep $DELAY
done
http://git-wip-us.apache.org/repos/asf/metron/blob/81282de2/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub
--
diff --git
a/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub
b/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub
index 3123782..c60c002 100644
--- a/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub
+++ b/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub
@@ -21,14 +21,20 @@
# simulates the behavior of a sensor by sending canned telemetry data
# to a Kafka topic.
#
-# a subset of the canned data is randomly selected and is sent in
+# a subset of the canned data is randomly selected and is sent in
# batches. the timestamp of the message is altered to match current
-# system time. the number of messages sent in each batch, along with
+# system time. the number of messages sent in each batch, along with
# the time delay between batches can be configured.
#
# start-snort-stub
#
+METRON_SYSCONFIG="/etc/default/metron"
+if [ -f "$METRON_SYSCONFIG" ]; then
+ set -a
+ . "$METRON_SYSCONFIG"
+fi
+
#
# how long to delay between each 'batch' in seconds.
#
@@ -36,20 +42,27 @@ DELAY=${1:-{{ sensor_stubs_delay }}}
#
# how many messages to send in each 'batch'. the messages are drawn randomly
-# from the entire set of canned data.
+# from the entire set of canned data.
#
COUNT=${2:-{{ sensor_stubs_count }}}
INPUT="{{ sensor_stubs_data }}/snort.out"
PRODUCER="{{ kafka_home }}/bin/kafka-console-producer.sh"
[40/50] [abbrv] metron git commit: METRON-1631 Alerts UI: Dash score does not show if only filtering by one group (sardell via merrimanr) closes apache/metron#1079
METRON-1631 Alerts UI: Dash score does not show if only filtering by one group
(sardell via merrimanr) closes apache/metron#1079
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/be9a9219
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/be9a9219
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/be9a9219
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: be9a9219875f8c6a2b0c07aa7b2d62484983d932
Parents: 55d46b0
Author: sardell
Authored: Fri Jun 29 09:49:24 2018 -0500
Committer: rmerriman
Committed: Fri Jun 29 09:49:24 2018 -0500
--
.../src/app/alerts/alerts-list/alerts-list.component.html| 5 +++--
.../src/app/alerts/alerts-list/alerts-list.component.ts | 2 ++
.../app/alerts/alerts-list/tree-view/tree-view.component.ts | 8 ++--
3 files changed, 7 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/be9a9219/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.html
--
diff --git
a/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.html
b/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.html
index 77f24d5..2a1802c 100644
---
a/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.html
+++
b/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.html
@@ -80,7 +80,7 @@
-http://git-wip-us.apache.org/repos/asf/metron/blob/be9a9219/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
b/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
index ca1bd50..15a87a4 100644
---
a/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
+++
b/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
@@ -77,6 +77,7 @@ export class AlertsListComponent implements OnInit, OnDestroy
{
groupFacets: Facets;
globalConfig: {} = {};
configSubscription: Subscription;
+ groups = [];
constructor(private router: Router,
private searchService: SearchService,
@@ -239,6 +240,7 @@ export class AlertsListComponent implements OnInit,
OnDestroy {
}
onGroupsChange(groups) {
+this.groups = groups;
this.queryBuilder.setGroupby(groups);
this.search();
}
http://git-wip-us.apache.org/repos/asf/metron/blob/be9a9219/metron-interface/metron-alerts/src/app/alerts/alerts-list/tree-view/tree-view.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/alerts/alerts-list/tree-view/tree-view.component.ts
b/metron-interface/metron-alerts/src/app/alerts/alerts-list/tree-view/tree-view.component.ts
index 9a7c78c..7b2e62c 100644
---
a/metron-interface/metron-alerts/src/app/alerts/alerts-list/tree-view/tree-view.component.ts
+++
b/metron-interface/metron-alerts/src/app/alerts/alerts-list/tree-view/tree-view.component.ts
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-import { Component, OnInit, OnChanges, SimpleChanges, OnDestroy } from
'@angular/core';
+import { Component, OnInit, OnChanges, SimpleChanges, OnDestroy, Input } from
'@angular/core';
import {Router} from '@angular/router';
import {Subscription, Observable} from 'rxjs/Rx';
@@ -47,12 +47,12 @@ import { GlobalConfigService } from
'../../../service/global-config.service';
export class TreeViewComponent extends TableViewComponent implements OnInit,
OnChanges, OnDestroy {
+ @Input() globalConfig: {} = {};
groupByFields: string[] = [];
topGroups: TreeGroupData[] = [];
groupResponse: GroupResponse = new GroupResponse();
treeGroupSubscriptionMap: {[key: string]: TreeAlertsSubscription } = {};
alertsChangedSubscription: Subscription;
- globalConfig: {} = {};
configSubscription: Subscription;
constructor(router: Router,
@@ -186,14 +186,10 @@ export class TreeViewComponent extends TableViewComponent
implements OnInit, OnC
ngOnInit() {
this.addAlertChangedListner();
-this.configSubscription =
this.globalConfigService.get().subscribe((config: {}) => {
- this.globalConfig = config;
-});
}
ngOnDestroy(): void {
this.removeAlertChangedLister();
-this.configSubscription.unsubscribe();
}
searchGroup(selectedGroup: TreeGroupData, searchRequest: SearchRequest):
Subscription {
[01/50] [abbrv] metron git commit: METRON-1540 Solr Integration tests should use actual schemas (justinleet) closes apache/metron#1005
Repository: metron
Updated Branches:
refs/heads/feature/METRON-1554-pcap-query-panel 9cee51eb2 -> f30e9cc1f
http://git-wip-us.apache.org/repos/asf/metron/blob/832296f0/metron-platform/metron-solr/src/test/resources/config/snort/conf/solrconfig.xml
--
diff --git
a/metron-platform/metron-solr/src/test/resources/config/snort/conf/solrconfig.xml
b/metron-platform/metron-solr/src/test/resources/config/snort/conf/solrconfig.xml
deleted file mode 100644
index fff9d84..000
---
a/metron-platform/metron-solr/src/test/resources/config/snort/conf/solrconfig.xml
+++ /dev/null
@@ -1,1601 +0,0 @@
-
-
-
-
-
-
-
-
- 7.2.0
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- ${solr.data.dir:}
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-${solr.lock.type:native}
-
-
-
-
-
-
-
-
-
-
-
-
- true
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- ${solr.ulog.dir:}
- ${solr.ulog.numVersionBuckets:65536}
-
-
-
-
- ${solr.autoCommit.maxTime:15000}
- false
-
-
-
-
-
- ${solr.autoSoftCommit.maxTime:-1}
-
-
-
-
-
-
-
-
-
-
-
-
-
-1024
-
-
-
--1
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-true
-
-
-
-
-
- 20
-
-
- 200
-
-
-
-
-
-
-
-
-
-
-
- static firstSearcher warming in solrconfig.xml
-
-
-
-
-
-false
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- explicit
- 10
-
-
-
- false
-
-
-
-
-
-
-
-
-
-
-
-
-
- explicit
- json
- true
- text
-
-
-
-
-
-
- explicit
-
-
- velocity
- browse
- layout
- Solritas
-
-
- edismax
-
- text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
- title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0
-
- 100%
- *:*
- 10
- *,score
-
-
- text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
- title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0
-
- text,features,name,sku,id,manu,cat,title,description,keywords,author,resourcename
- 3
-
-
- on
- true
- cat
- manu_exact
- content_type
- author_s
- ipod
- GB
- 1
- cat,inStock
- after
- price
- 0
- 600
- 50
- popularity
- 0
- 10
- 3
- manufacturedate_dt
- NOW/YEAR-10YEARS
- NOW
- +1YEAR
- before
- after
-
-
- on
- content features title name
- true
- html
- b
- /b
- 0
- title
- 0
- name
- 3
- 200
- content
- 750
-
-
- on
- false
- 5
- 2
- 5
- true
- true
- 5
- 3
-
-
-
-
- spellcheck
-
-
-
-
-
-
- text
-
-
-
-
-
-
-
-
- true
- ignored_
-
-
- true
- links
- ignored_
-
-
-
-
-
-
-
-
-text_general
-
-
-
-
-
- default
- text
- solr.DirectSolrSpellChecker
-
- internal
-
- 0.5
-
- 2
-
- 1
-
- 5
-
- 4
-
- 0.01
-
-
-
-
-
- wordbreak
- solr.WordBreakSolrSpellChecker
- name
- true
- true
- 10
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- default
- wordbreak
- on
- true
- 10
- 5
- 5
- true
- true
- 10
- 5
-
-
- spellcheck
-
-
-
-
-
-
- mySuggester
- FuzzyLookupFactory
- DocumentDictionaryFactory
- cat
- price
- string
- false
-
-
-
-
-
- true
- 10
-
-
- suggest
-
-
-
-
-
-
-
-
-
-
- true
-
-
- tvComponent
-
-
-
-
-
-
-
-
- lingo3g
- true
- com.carrotsearch.lingo3g.Lingo3GClusteringAlgorithm
- clustering/carrot2
-
-
-
- lingo
- org.carrot2.clustering.lingo.LingoClusteringAlgorithm
- clustering/carrot2
-
-
-
- stc
- org.carrot2.clustering.stc.STCClusteringAlgorithm
- clustering/carrot2
-
-
-
- kmeans
- org.carrot2.clustering.kmeans.BisectingKMeansClusteringAlgorithm
- clustering/carrot2
-
-
-
-
-
-
- true
-
[05/50] [abbrv] metron git commit: Merge branch 'master' into feature/METRON-1416-upgrade-solr
Merge branch 'master' into feature/METRON-1416-upgrade-solr Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/eb336665 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/eb336665 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/eb336665 Branch: refs/heads/feature/METRON-1554-pcap-query-panel Commit: eb3366656de643e7436f7cf280561b94f3921c65 Parents: bc16e68 71a015f Author: justinjleet Authored: Fri May 18 14:21:24 2018 -0400 Committer: justinjleet Committed: Fri May 18 14:21:24 2018 -0400 -- dependencies_with_url.csv | 56 ++- .../metron/profiler/ProfileMeasurement.java | 3 +- .../apache/metron/profiler/ProfilePeriod.java | 3 +- .../metron/profiler/ProfileMeasurementTest.java | 108 + .../metron/profiler/ProfilePeriodTest.java | 49 +++ metron-analytics/metron-profiler/README.md | 62 ++- .../src/main/config/profiler.properties | 13 + .../src/main/flux/profiler/remote.yaml | 3 + .../zookeeper/profile-with-stats/profiler.json | 12 + .../integration/ProfilerIntegrationTest.java| 83 +++- .../metron/statistics/StatisticsProvider.java | 6 +- .../statistics/sampling/UniformSampler.java | 5 +- metron-deployment/amazon-ec2/ansible.cfg| 5 +- .../roles/ambari_config/vars/single_node_vm.yml | 1 + metron-deployment/packaging/ambari/.gitignore | 2 + .../enrichment-splitjoin.properties.j2 | 63 --- .../templates/enrichment-unified.properties.j2 | 60 --- .../package/templates/profiler.properties.j2| 13 + metron-interface/metron-alerts/README.md| 6 + metron-interface/metron-alerts/proxy.conf.json | 9 - .../alert-details/alert-details.component.ts| 2 +- .../alert-details/alerts-details.routing.ts | 2 +- .../alerts/alerts-list/alerts-list.component.ts | 25 +- .../table-view/table-view.component.ts | 32 +- .../tree-view/tree-view.component.ts| 17 +- .../alerts/meta-alerts/meta-alerts.component.ts | 20 +- .../metron-alerts/src/app/app.module.ts | 4 +- .../src/app/model/search-request.ts | 2 +- .../src/app/service/authentication.service.ts | 10 +- .../service/elasticsearch-localstorage-impl.ts | 6 +- .../src/app/service/global-config.service.ts| 50 +++ .../src/app/service/update.service.ts | 10 +- .../metron-alerts/src/app/utils/constants.ts| 4 +- .../metron-alerts/src/app/utils/utils.ts| 7 +- metron-interface/metron-config/proxy.conf.json | 4 +- metron-interface/metron-rest/README.md | 2 + metron-interface/metron-rest/pom.xml| 17 +- .../metron/rest/MetronRestApplication.java | 7 + .../apache/metron/rest/MetronRestConstants.java | 2 + .../metron/rest/config/JpaConfiguration.java| 6 +- .../metron/rest/config/WebSecurityConfig.java | 19 +- .../rest/service/impl/SearchServiceImpl.java| 42 +- .../src/main/resources/application-test.yml | 2 +- .../src/main/resources/application.yml | 2 +- .../metron-rest/src/main/scripts/metron-rest.sh | 2 +- .../SearchControllerIntegrationTest.java| 9 +- .../service/impl/SearchServiceImplTest.java | 60 ++- metron-platform/metron-common/README.md | 15 +- .../org/apache/metron/common/Constants.java | 10 +- .../configuration/SensorParserConfig.java | 410 +++ .../configuration/profiler/ProfileResult.java | 4 +- .../profiler/ProfileResultExpressions.java | 4 +- .../profiler/ProfileTriageExpressions.java | 3 +- .../configuration/profiler/ProfilerConfig.java | 12 +- .../common/utils/CompressionStrategies.java | 100 + .../common/utils/CompressionStrategy.java | 52 +++ .../apache/metron/common/utils/SerDeUtils.java | 7 +- .../profiler/ProfilerConfigTest.java| 85 +++- .../common/utils/CompressionUtilsTest.java | 62 +++ .../nonbulk/geo/GeoEnrichmentLoader.java| 70 +++- .../nonbulk/geo/GeoEnrichmentLoaderTest.java| 40 +- .../parsers/topology/ParserTopologyBuilder.java | 139 --- .../parsers/topology/ParserTopologyCLI.java | 147 +-- .../components/ParserTopologyComponent.java | 80 ++-- .../parsers/topology/ParserTopologyCLITest.java | 122 -- ...pleHbaseEnrichmentWriterIntegrationTest.java | 69 ++-- .../integration/WriterBoltIntegrationTest.java | 382 +++-- .../apache/metron/writer/kafka/KafkaWriter.java | 5 + .../stellar/common/utils/BloomFilter.java | 8 +- .../metron/stellar/common/utils/SerDeUtils.java | 16 +- .../common/utils/StellarProcessorUtils.java | 101 - .../metron/stellar/dsl/functions/Ordinal.java | 36 ++ .../stellar/dsl/functions/OrdinalFunctions.java | 48 ++- .../dsl/functions/OrdinalFunctionsTest.java
metron git commit: METRON-1641: Enable Pcap jobs to be submitted asynchronously (mmiklavc via mmiklavc) closes apache/metron#1081
Repository: metron
Updated Branches:
refs/heads/feature/METRON-1554-pcap-query-panel 4e586ac55 -> 9cee51eb2
METRON-1641: Enable Pcap jobs to be submitted asynchronously (mmiklavc via
mmiklavc) closes apache/metron#1081
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9cee51eb
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9cee51eb
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9cee51eb
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 9cee51eb2c428eb7bf3aa791f5ec1293ed36c43a
Parents: 4e586ac
Author: mmiklavc
Authored: Tue Jul 10 19:24:58 2018 -0600
Committer: Michael Miklavcic
Committed: Tue Jul 10 19:24:58 2018 -0600
--
.../rest/service/impl/PcapServiceImpl.java | 11 +-
.../apache/metron/rest/mock/MockPcapJob.java| 15 +-
.../apache/metron/common/utils/HDFSUtils.java | 39 +++-
.../metron/common/utils/HDFSUtilsTest.java | 59 +
metron-platform/metron-job/pom.xml | 39
.../java/org/apache/metron/job/JobStatus.java | 83 +++
.../java/org/apache/metron/job/Pageable.java| 38
.../java/org/apache/metron/job/Statusable.java | 56 +
.../org/apache/metron/pcap/query/PcapCli.java | 39 +---
.../apache/metron/pcap/query/ResultsWriter.java | 48
.../org/apache/metron/pcap/PcapJobTest.java | 198
.../PcapTopologyIntegrationTest.java| 1 -
.../apache/metron/pcap/query/PcapCliTest.java | 54 +++--
metron-platform/metron-pcap/pom.xml | 7 +-
.../java/org/apache/metron/pcap/PcapFiles.java | 42
.../java/org/apache/metron/pcap/mr/PcapJob.java | 225 +++
.../metron/pcap/writer/ResultsWriter.java | 59 +
metron-platform/pom.xml | 1 +
18 files changed, 848 insertions(+), 166 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/9cee51eb/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java
--
diff --git
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java
index 4dae1e5..dd4af5c 100644
---
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java
+++
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java
@@ -17,6 +17,11 @@
*/
package org.apache.metron.rest.service.impl;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -35,12 +40,6 @@ import
org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Service;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
@Service
public class PcapServiceImpl implements PcapService {
http://git-wip-us.apache.org/repos/asf/metron/blob/9cee51eb/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapJob.java
--
diff --git
a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapJob.java
b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapJob.java
index 3aa9ce3..a7eca31 100644
---
a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapJob.java
+++
b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapJob.java
@@ -17,6 +17,12 @@
*/
package org.apache.metron.rest.mock;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -24,15 +30,6 @@ import org.apache.metron.common.hadoop.SequenceFileIterable;
import org.apache.metron.pcap.filter.PcapFilterConfigurator;
import org.apache.metron.pcap.mr.PcapJob;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import static org.mockito.Matchers.anyList;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
public class MockPcapJob extends PcapJob {
private String basePath;
metron git commit: METRON-1655 Make REGEXP_MATCH take multiple regexs in the 2nd arg (ottobackwards) closes apache/metron#1098
Repository: metron
Updated Branches:
refs/heads/master f183d987a -> b2a22b81a
METRON-1655 Make REGEXP_MATCH take multiple regexs in the 2nd arg
(ottobackwards) closes apache/metron#1098
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b2a22b81
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b2a22b81
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b2a22b81
Branch: refs/heads/master
Commit: b2a22b81a3d763be81be0746b608fecf2e09c04e
Parents: f183d98
Author: ottobackwards
Authored: Tue Jul 10 13:28:39 2018 -0400
Committer: otto
Committed: Tue Jul 10 13:28:39 2018 -0400
--
metron-stellar/stellar-common/README.md | 4 +--
.../stellar/dsl/functions/RegExFunctions.java | 27 +++-
.../dsl/functions/RegExFunctionsTest.java | 7 +
3 files changed, 30 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/b2a22b81/metron-stellar/stellar-common/README.md
--
diff --git a/metron-stellar/stellar-common/README.md
b/metron-stellar/stellar-common/README.md
index 0dd8e16..88a3834 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -886,10 +886,10 @@ Where:
* Returns: The reduction of the list.
### `REGEXP_MATCH`
- * Description: Determines whether a regex matches a string
+ * Description: Determines whether a regex matches a string. If a list of
patterns is passed, then the matching is an OR operation
* Input:
* string - The string to test
-* pattern - The proposed regex pattern
+* pattern - The proposed regex pattern or a list of patterns
* Returns: True if the regex pattern matches the string and false if
otherwise.
### `REGEXP_GROUP_VAL`
http://git-wip-us.apache.org/repos/asf/metron/blob/b2a22b81/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RegExFunctions.java
--
diff --git
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RegExFunctions.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RegExFunctions.java
index ddc8860..48e82a3 100644
---
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RegExFunctions.java
+++
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RegExFunctions.java
@@ -30,10 +30,10 @@ import org.apache.metron.stellar.dsl.Stellar;
public class RegExFunctions {
@Stellar(name = "REGEXP_MATCH",
- description = "Determines whether a regex matches a string",
+ description = "Determines whether a regex matches a string, if a list of
patterns is passed, then the matching is an OR operation",
params = {
"string - The string to test",
- "pattern - The proposed regex pattern"
+ "pattern - The proposed regex pattern or a list of proposed regex
patterns"
},
returns = "True if the regex pattern matches the string and false if
otherwise.")
public static class RegexpMatch extends BaseStellarFunction {
@@ -42,14 +42,29 @@ public class RegExFunctions {
public Object apply(List list) {
if (list.size() < 2) {
throw new IllegalStateException(
-"REGEXP_MATCH expects two args: [string, pattern] where pattern is
a regexp pattern");
+"REGEXP_MATCH expects two args: [string, pattern] where pattern is
a regexp pattern or a list of regexp patterns");
}
- String patternString = (String) list.get(1);
+ Object patternObject = list.get(1);
String str = (String) list.get(0);
- if (str == null || patternString == null) {
+ if (str == null || patternObject == null) {
return false;
}
- return
PatternCache.INSTANCE.getPattern(patternString).matcher(str).matches();
+ if (patternObject instanceof String) {
+return
PatternCache.INSTANCE.getPattern((String)patternObject).matcher(str).matches();
+ } else if (patternObject instanceof Iterable) {
+boolean matches = false;
+for (Object thisPatternObject : (Iterable)patternObject) {
+ if (thisPatternObject == null) {
+continue;
+ }
+ if
(PatternCache.INSTANCE.getPattern(thisPatternObject.toString()).matcher(str).matches())
{
+matches = true;
+break;
+ }
+}
+return matches;
+ }
+ return false;
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/b2a22b81/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/RegExFunctionsTest.java
