Merge branch 'master' of http://github.com/apache/metron into feature/METRON-1090-stellar-assignment
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f548b191 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f548b191 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f548b191 Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: f548b191b8348e14825fd9ec4bff3703193be51d Parents: 3df9498 3c4c069 Author: otto <o...@apache.org> Authored: Sun Nov 18 13:11:19 2018 -0500 Committer: otto <o...@apache.org> Committed: Sun Nov 18 13:11:19 2018 -0500 ---------------------------------------------------------------------- Upgrading.md | 24 + dependencies_with_url.csv | 4 + .../committer-utils/metron-committer-common | 358 + dev-utilities/committer-utils/prepare-commit | 241 +- metron-analytics/metron-maas-common/pom.xml | 2 +- metron-deployment/Kerberos-manual-setup.md | 154 +- metron-deployment/development/README.md | 34 + .../development/knox-demo-ldap.ldif | 101 + .../configuration/metron-enrichment-env.xml | 8 +- .../METRON/CURRENT/configuration/metron-env.xml | 9 - .../CURRENT/configuration/metron-rest-env.xml | 10 +- .../configuration/metron-security-env.xml | 186 + .../common-services/METRON/CURRENT/metainfo.xml | 3 + .../package/files/metaalert_index.template | 4 +- .../CURRENT/package/scripts/metron_service.py | 2 - .../package/scripts/params/params_linux.py | 30 +- .../CURRENT/package/scripts/rest_commands.py | 4 + .../METRON/CURRENT/package/templates/metron.j2 | 16 + .../METRON/CURRENT/themes/metron_theme.json | 181 +- .../docker/rpm-docker/SPECS/metron.spec | 6 +- metron-interface/metron-alerts/cypress.json | 7 + .../metron-alerts/cypress/fixtures/config.json | 23 + .../cypress/fixtures/pcap.page-01.json | 12383 +++++++++++ .../cypress/fixtures/pcap.status-00.json | 7 + .../cypress/fixtures/pcap.status-01.json | 7 + .../cypress/fixtures/pcap.status-02.json | 7 + .../metron-alerts/cypress/fixtures/search.json | 5647 +++++ .../cypress/integration/pcap/pcap.spec.js | 228 + .../metron-alerts/package-lock.json | 1134 + metron-interface/metron-alerts/package.json | 11 +- .../alert-details/alert-details.component.ts | 10 +- .../alerts/alerts-list/alerts-list.component.ts | 9 +- .../table-view/table-view.component.spec.ts | 13 +- .../table-view/table-view.component.ts | 49 +- .../tree-view/tree-view.component.spec.ts | 13 +- .../tree-view/tree-view.component.ts | 20 +- .../alerts/meta-alerts/meta-alerts.component.ts | 2 - .../save-search/save-search.component.spec.ts | 4 +- .../alerts/save-search/save-search.component.ts | 12 +- .../saved-searches.component.spec.ts | 5 +- .../saved-searches/saved-searches.component.ts | 41 +- .../metron-alerts/src/app/app.component.html | 1 + .../metron-alerts/src/app/app.component.spec.ts | 4 + .../metron-alerts/src/app/app.module.ts | 21 +- .../src/app/model/confirmation-type.ts | 21 + .../metron-alerts/src/app/model/dialog-type.ts | 21 + .../pcap-panel/pcap-panel.component.spec.ts | 9 + .../app/pcap/pcap-panel/pcap-panel.component.ts | 68 +- .../src/app/pcap/service/pcap.service.spec.ts | 13 +- .../src/app/pcap/service/pcap.service.ts | 17 +- .../src/app/service/alerts.service.ts | 5 +- .../src/app/service/app-config.service.ts | 41 + .../src/app/service/authentication.service.ts | 10 +- .../src/app/service/column-names.service.ts | 4 - .../src/app/service/dialog.service.spec.ts | 78 + .../src/app/service/dialog.service.ts | 55 + .../src/app/service/global-config.service.ts | 5 +- .../src/app/service/meta-alert.service.ts | 11 +- .../src/app/service/search.service.ts | 11 +- .../src/app/service/update.service.ts | 9 +- .../src/app/shared/metron-dialog-box.ts | 91 - .../metron-dialog/metron-dialog.component.html | 40 + .../metron-dialog/metron-dialog.component.scss | 20 + .../metron-dialog.component.spec.ts | 127 + .../metron-dialog/metron-dialog.component.ts | 66 + .../metron-alerts/src/assets/app-config.json | 3 + metron-interface/metron-config/angular-cli.json | 51 - metron-interface/metron-config/angular.json | 155 + metron-interface/metron-config/karma.conf.js | 60 +- .../metron-config/package-lock.json | 18987 ++++++----------- metron-interface/metron-config/package.json | 70 +- metron-interface/metron-config/pom.xml | 10 +- .../scripts/prepend_license_header.sh | 2 +- .../metron-config/scripts/start_dev.sh | 2 +- .../metron-config/src/app/_fonts.scss | 101 + .../metron-config/src/app/_main.scss | 112 - .../metron-config/src/app/app.component.spec.ts | 64 +- .../metron-config/src/app/app.component.ts | 3 +- .../metron-config/src/app/app.config.ts | 4 +- .../metron-config/src/app/app.module.ts | 17 +- .../metron-config/src/app/app.routes.ts | 10 +- .../general-settings.component.spec.ts | 85 +- .../general-settings.component.ts | 48 +- .../metron-config/src/app/global.scss | 2 +- .../default-headers.interceptor.ts | 32 + .../src/app/login/login.component.spec.ts | 42 +- .../metron-config/src/app/model/rest-error.ts | 4 +- .../src/app/navbar/navbar.component.spec.ts | 19 +- .../src/app/navbar/navbar.component.ts | 31 +- .../metron-config/src/app/navbar/navbar.html | 2 +- .../metron-config/src/app/rxjs-operators.ts | 32 - .../sensor-field-schema.component.html | 2 +- .../sensor-field-schema.component.spec.ts | 1088 +- .../sensor-grok/sensor-grok.component.spec.ts | 137 +- .../sensor-grok/sensor-grok.component.ts | 2 +- ...sor-parser-config-readonly.component.spec.ts | 430 +- .../sensor-parser-config-readonly.component.ts | 389 +- .../sensor-parser-config.component.spec.ts | 615 +- .../sensor-parser-config.component.ts | 442 +- .../sensor-parser-list.component.spec.ts | 512 +- .../sensor-raw-json.component.spec.ts | 345 +- .../sensor-storm-settings.component.spec.ts | 294 +- .../sensor-threat-triage.component.spec.ts | 129 +- .../app/service/authentication.service.spec.ts | 276 +- .../src/app/service/authentication.service.ts | 73 +- .../app/service/global-config.service.spec.ts | 101 +- .../src/app/service/global-config.service.ts | 25 +- .../app/service/grok-validation.service.spec.ts | 122 +- .../src/app/service/grok-validation.service.ts | 53 +- .../src/app/service/hdfs.service.spec.ts | 129 +- .../src/app/service/hdfs.service.ts | 63 +- .../src/app/service/kafka.service.spec.ts | 123 +- .../src/app/service/kafka.service.ts | 57 +- .../sensor-enrichment-config.service.spec.ts | 240 +- .../service/sensor-enrichment-config.service.ts | 76 +- .../sensor-indexing-config.service.spec.ts | 150 +- .../service/sensor-indexing-config.service.ts | 62 +- ...sensor-parser-config-history.service.spec.ts | 107 +- .../sensor-parser-config-history.service.ts | 55 +- .../sensor-parser-config.service.spec.ts | 206 +- .../app/service/sensor-parser-config.service.ts | 130 +- .../src/app/service/stellar.service.spec.ts | 187 +- .../src/app/service/stellar.service.ts | 72 +- .../src/app/service/storm.service.spec.ts | 402 +- .../src/app/service/storm.service.ts | 156 +- .../shared/ace-editor/ace-editor.component.ts | 9 +- .../metron-config/src/app/shared/auth-guard.ts | 2 +- .../src/app/shared/metron-dialog-box.ts | 6 +- .../sample-data/sample-data.component.html | 2 +- .../sample-data/sample-data.component.spec.ts | 41 +- .../metron-config/src/app/util/httpUtil.ts | 24 +- .../src/app/util/httpUtils.spec.ts | 57 +- .../verticalnavbar.component.spec.ts | 30 +- .../verticalnavbar/verticalnavbar.component.ts | 3 - .../src/app/verticalnavbar/verticalnavbar.html | 4 +- metron-interface/metron-config/src/styles.scss | 2 +- metron-interface/metron-config/src/test.ts | 66 +- .../metron-config/src/tsconfig.app.json | 23 + .../metron-config/src/tsconfig.json | 24 - .../metron-config/src/tsconfig.spec.json | 24 + metron-interface/metron-config/src/typings.d.ts | 2 +- metron-interface/metron-config/tsconfig.json | 20 + metron-interface/metron-rest/README.md | 38 + metron-interface/metron-rest/pom.xml | 12 + .../src/main/config/rest_application.yml | 32 +- .../apache/metron/rest/MetronRestConstants.java | 1 + .../metron/rest/config/WebSecurityConfig.java | 73 +- .../metron/rest/controller/UserController.java | 13 + .../rest/service/impl/MetaAlertServiceImpl.java | 2 +- .../src/main/resources/application-vagrant.yml | 15 +- .../metron-rest/src/main/scripts/metron-rest.sh | 28 +- metron-platform/Performance-tuning-guide.md | 6 +- metron-platform/elasticsearch-shaded/pom.xml | 47 +- .../META-INF/log4j-provider.properties | 18 - metron-platform/metron-common/README.md | 48 +- .../src/main/config/zookeeper/global.json | 1 - .../common/bolt/ConfiguredEnrichmentBolt.java | 14 + .../common/bolt/ConfiguredParserBolt.java | 13 + .../common/configuration/ConfigOption.java | 7 + metron-platform/metron-data-management/pom.xml | 5 +- metron-platform/metron-elasticsearch/README.md | 45 +- metron-platform/metron-elasticsearch/pom.xml | 32 +- .../client/ElasticsearchClient.java | 245 + .../client/ElasticsearchClientFactory.java | 189 + .../config/ElasticsearchClientConfig.java | 187 + .../config/ElasticsearchClientOptions.java | 60 + .../dao/ElasticsearchColumnMetadataDao.java | 101 +- .../elasticsearch/dao/ElasticsearchDao.java | 21 +- .../dao/ElasticsearchMetaAlertDao.java | 2 +- .../dao/ElasticsearchMetaAlertSearchDao.java | 6 +- .../dao/ElasticsearchMetaAlertUpdateDao.java | 4 +- .../dao/ElasticsearchRequestSubmitter.java | 13 +- .../dao/ElasticsearchRetrieveLatestDao.java | 28 +- .../dao/ElasticsearchSearchDao.java | 19 +- .../dao/ElasticsearchUpdateDao.java | 19 +- .../elasticsearch/utils/ElasticsearchUtils.java | 182 +- .../elasticsearch/utils/FieldMapping.java | 32 + .../elasticsearch/utils/FieldProperties.java | 36 + .../writer/ElasticsearchWriter.java | 26 +- .../dao/ElasticsearchColumnMetadataDaoTest.java | 59 +- .../elasticsearch/dao/ElasticsearchDaoTest.java | 8 +- .../dao/ElasticsearchRequestSubmitterTest.java | 23 +- .../dao/ElasticsearchUpdateDaoTest.java | 12 +- .../ElasticsearchMetaAlertIntegrationTest.java | 9 +- .../ElasticsearchSearchIntegrationTest.java | 145 +- .../ElasticsearchUpdateIntegrationTest.java | 2 +- .../components/ElasticSearchComponent.java | 6 +- metron-platform/metron-enrichment/README.md | 43 +- .../enrichment/bolt/GenericEnrichmentBolt.java | 1 + .../enrichment/bolt/ThreatIntelJoinBolt.java | 6 - .../main/scripts/start_enrichment_topology.sh | 4 +- .../dao/metaalert/MetaAlertSearchDao.java | 4 +- .../dao/metaalert/MetaAlertIntegrationTest.java | 2 +- .../src/main/config/zookeeper/global.json | 2 +- .../parsers/fireeye/BasicFireEyeParser.java | 347 +- .../paloalto/BasicPaloAltoFirewallParser.java | 413 +- .../apache/metron/parsers/paloalto/README.md | 32 + .../BasicPaloAltoFirewallParserTest.java | 244 + metron-stellar/stellar-common/README.md | 98 +- metron-stellar/stellar-common/pom.xml | 40 + .../stellar/common/shell/cli/StellarShell.java | 2 + .../org/apache/metron/stellar/dsl/Context.java | 5 + .../metron/stellar/dsl/StellarFunction.java | 9 +- .../metron/stellar/dsl/StellarFunctions.java | 5 + .../stellar/dsl/functions/RestConfig.java | 166 + .../stellar/dsl/functions/RestFunctions.java | 388 + .../resolver/BaseFunctionResolver.java | 44 + .../functions/resolver/FunctionResolver.java | 14 +- .../stellar/dsl/functions/BasicStellarTest.java | 20 +- .../dsl/functions/DateFunctionsTest.java | 12 +- .../dsl/functions/RestFunctionsTest.java | 601 + .../resolver/BaseFunctionResolverTest.java | 169 + pom.xml | 1 + 213 files changed, 35599 insertions(+), 18251 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/f548b191/metron-stellar/stellar-common/README.md ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/f548b191/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java ---------------------------------------------------------------------- diff --cc metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java index 28f1ae4,79f97bc..007ee72 --- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java @@@ -18,11 -18,20 +18,22 @@@ package org.apache.metron.stellar.dsl.functions; + import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run; + import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.runPredicate; + import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.validate; + import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import java.text.DecimalFormat; + import java.util.ArrayList; + import java.util.Arrays; + import java.util.Collection; + import java.util.HashMap; + import java.util.HashSet; + import java.util.List; + import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.apache.metron.stellar.common.StellarProcessor; import org.apache.metron.stellar.dsl.Context;
