otto(o...@apache.org) syncing feature/METRON-1136-extensions-parsers with master
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/43646650 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/43646650 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/43646650 Branch: refs/heads/feature/METRON-1136-extensions-parsers Commit: 4364665002756c1193cf834f3a2f53d774122760 Parents: ffcb91e Author: ottofowler <> Authored: Wed Aug 30 11:07:03 2017 -0400 Committer: otto <o...@apache.org> Committed: Wed Aug 30 11:07:03 2017 -0400 ---------------------------------------------------------------------- .gitignore | 2 +- metron-analytics/metron-maas-common/pom.xml | 11 + .../metron/maas/functions/MaaSFunctions.java | 324 +++++++++++++++++++ metron-analytics/metron-maas-service/README.md | 21 +- metron-analytics/metron-maas-service/pom.xml | 11 - .../metron/maas/functions/MaaSFunctions.java | 324 ------------------- .../org/apache/metron/maas/service/Client.java | 15 + .../apache/metron/maas/service/Constants.java | 5 +- .../metron/maas/service/runner/Runner.java | 9 +- .../metron/maas/submit/ModelSubmission.java | 5 + .../METRON/CURRENT/role_command_order.json | 2 + .../configuration/metron-indexing-env.xml | 2 +- .../configuration/metron-profiler-env.xml | 155 +++++++++ .../common-services/METRON/CURRENT/metainfo.xml | 43 +++ .../package/scripts/params/params_linux.py | 25 ++ .../package/scripts/params/status_params.py | 11 + .../package/scripts/profiler_commands.py | 194 +++++++++++ .../CURRENT/package/scripts/profiler_master.py | 94 ++++++ .../package/templates/profiler.properties.j2 | 47 +++ .../METRON/CURRENT/service_advisor.py | 5 + .../METRON/CURRENT/themes/metron_theme.json | 198 +++++++++++- .../roles/ambari_config/vars/single_node_vm.yml | 7 +- .../roles/ambari_config/vars/small_cluster.yml | 5 +- metron-interface/metron-alerts/README.md | 22 +- .../metron-alerts/alerts-server-e2e.js | 31 +- .../e2e/alerts-list/alerts-list.e2e-spec.ts | 17 +- .../configure-table/configure-table.e2e-spec.ts | 15 +- .../save-search/save-search.e2e-spec.ts | 11 + .../metron-alerts/e2e/login/login.e2e-spec.ts | 44 +++ .../metron-alerts/e2e/login/login.po.ts | 65 ++++ .../metron-alerts/e2e/utils/e2e_util.ts | 30 ++ .../metron-alerts/protractor.conf.js | 13 +- metron-interface/metron-alerts/proxy.conf.json | 4 +- .../metron-alerts/scripts/alerts-server.js | 12 +- .../scripts/start-server-for-e2e.sh | 3 +- .../metron-alerts/src/_variables.scss | 3 + .../alert-details/alert-details.component.html | 4 +- .../alert-details/alert-details.component.ts | 44 ++- .../alert-details/alerts-details.routing.ts | 2 +- .../alerts-list/alerts-list.component.html | 6 +- .../alerts/alerts-list/alerts-list.component.ts | 32 +- .../alerts/alerts-list/alerts-list.module.ts | 2 +- .../src/app/alerts/alerts-list/query-builder.ts | 18 +- .../metron-alerts/src/app/app-routing.module.ts | 12 +- .../metron-alerts/src/app/app.component.html | 5 +- .../metron-alerts/src/app/app.component.scss | 12 + .../metron-alerts/src/app/app.component.ts | 8 + .../metron-alerts/src/app/app.module.ts | 14 +- .../src/app/login/login.component.html | 28 ++ .../src/app/login/login.component.scss | 55 ++++ .../src/app/login/login.component.spec.ts | 65 ++++ .../src/app/login/login.component.ts | 43 +++ .../metron-alerts/src/app/login/login.module.ts | 28 ++ .../src/app/login/login.routing.ts | 25 ++ .../metron-alerts/src/app/model/alert-source.ts | 52 +++ .../metron-alerts/src/app/model/alert.ts | 45 +-- .../src/app/model/search-request.ts | 12 +- .../metron-alerts/src/app/model/sort-field.ts | 21 ++ .../src/app/service/alert.service.ts | 13 +- .../src/app/service/authentication.service.ts | 91 ++++++ .../src/app/service/data-source.ts | 5 +- .../service/elasticsearch-localstorage-impl.ts | 11 +- .../src/app/service/rest-api-impl.ts | 46 +++ .../metron-alerts/src/app/shared/auth-guard.ts | 50 +++ .../metron-alerts/src/app/shared/login-guard.ts | 40 +++ .../src/app/utils/elasticsearch-utils.ts | 2 +- .../metron-alerts/src/app/utils/httpUtil.ts | 6 +- .../src/environments/environment.prod.ts | 3 +- .../transformation/StellarTransformation.java | 11 + .../StellarTransformationTest.java | 47 +++ metron-platform/metron-parsers/README.md | 43 +++ 71 files changed, 2177 insertions(+), 509 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 2d048d7..bcb3642 100644 --- a/.gitignore +++ b/.gitignore @@ -22,5 +22,5 @@ tmp/** tmp/**/* temp/** temp/**/* - +metron-interface/metron-alerts/node/ repodata/ http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-analytics/metron-maas-common/pom.xml ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-maas-common/pom.xml b/metron-analytics/metron-maas-common/pom.xml index 13fb7b9..6921e51 100644 --- a/metron-analytics/metron-maas-common/pom.xml +++ b/metron-analytics/metron-maas-common/pom.xml @@ -34,6 +34,17 @@ </properties> <dependencies> <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <exclusions> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-auth</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> <groupId>commons-cli</groupId> <artifactId>commons-cli</artifactId> <version>1.2</version> http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java new file mode 100644 index 0000000..eacb64d --- /dev/null +++ b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java @@ -0,0 +1,324 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.maas.functions; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import java.lang.invoke.MethodHandles; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.maas.config.Endpoint; +import org.apache.metron.maas.config.MaaSConfig; +import org.apache.metron.maas.config.ModelEndpoint; +import org.apache.metron.maas.discovery.ServiceDiscoverer; +import org.apache.metron.maas.util.ConfigUtil; +import org.apache.metron.maas.util.RESTUtil; +import org.apache.metron.stellar.common.utils.JSONUtils; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.dsl.StellarFunction; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MaaSFunctions { + protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static class ModelCacheKey { + String name; + String version; + String method; + Map<String, String> args; + public ModelCacheKey(String name, String version, String method, Map<String, String> args) { + this.name = name; + this.version = version; + this.method = method; + this.args = args; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ModelCacheKey that = (ModelCacheKey) o; + + if (name != null ? !name.equals(that.name) : that.name != null) return false; + if (version != null ? !version.equals(that.version) : that.version != null) return false; + if (method != null ? !method.equals(that.method) : that.method != null) return false; + return args != null ? args.equals(that.args) : that.args == null; + + } + + @Override + public int hashCode() { + int result = name != null ? name.hashCode() : 0; + result = 31 * result + (version != null ? version.hashCode() : 0); + result = 31 * result + (method != null ? method.hashCode() : 0); + result = 31 * result + (args != null ? args.hashCode() : 0); + return result; + } + } + + @Stellar(name="MODEL_APPLY" + , namespace="MAAS" + , description = "Returns the output of a model deployed via Model as a Service. NOTE: Results are cached locally for 10 minutes." + , params = { "endpoint - A map containing the name, version, and url for the REST endpoint" + , "function - The optional endpoint path; default is 'apply'" + , "model_args - A Dictionary of arguments for the model (these become request params)" + } + , returns = "The output of the model deployed as a REST endpoint in Map form. Assumes REST endpoint returns a JSON Map." + ) + public static class ModelApply implements StellarFunction { + private boolean isInitialized = false; + private ServiceDiscoverer discoverer; + private Cache<ModelCacheKey, Map<String, Object> > resultCache; + public ModelApply() { + resultCache = CacheBuilder.newBuilder() + .concurrencyLevel(4) + .weakKeys() + .maximumSize(100000) + .expireAfterWrite(10, TimeUnit.MINUTES) + .build(); + } + + @Override + public Object apply(List<Object> args, Context context) throws ParseException { + if(args.size() < 2) { + throw new ParseException("Unable to execute model_apply. " + + "Expected arguments: endpoint_map:map, " + + " [endpoint method:string], model_args:map" + ); + } + if(!isInitialized) { + return null; + } + int i = 0; + if(args.size() == 0) { + return null; + } + Object endpointObj = args.get(i++); + Map endpoint = null; + String modelName; + String modelVersion; + String modelUrl; + if(endpointObj instanceof Map) { + endpoint = (Map)endpointObj; + modelName = endpoint.get("name") + ""; + modelVersion = endpoint.get("version") + ""; + modelUrl = endpoint.get("url") + ""; + } + else { + return null; + } + String modelFunction = "apply"; + Map<String, String> modelArgs = new HashMap<>(); + if(args.get(i) instanceof String) { + String func = (String)args.get(i); + if(endpoint.containsKey("endpoint:" + func)) { + modelFunction = "" + endpoint.get("endpoint:" + func); + } + else { + modelFunction = func; + } + i++; + } + + if(args.get(i) instanceof Map) { + if(endpoint.containsKey("endpoint:apply")) { + modelFunction = "" + endpoint.get("endpoint:apply"); + } + modelArgs = (Map)args.get(i); + } + if( modelName == null + || modelVersion == null + || modelFunction == null + ) { + return null; + } + ModelCacheKey cacheKey = new ModelCacheKey(modelName, modelVersion, modelFunction, modelArgs); + Map<String, Object> ret = resultCache.getIfPresent(cacheKey); + if(ret != null) { + return ret; + } + else { + String url = modelUrl; + if (url.endsWith("/")) { + url = url.substring(0, url.length() - 1); + } + if (modelFunction.startsWith("/")) { + modelFunction = modelFunction.substring(1); + } + try { + URL u = new URL(url + "/" + modelFunction); + + String results = RESTUtil.INSTANCE.getRESTJSONResults(u, modelArgs); + ret = JSONUtils.INSTANCE.load(results, new TypeReference<Map<String, Object>>() { + }); + resultCache.put(cacheKey, ret); + return ret; + } catch (Exception e) { + LOG.error(e.getMessage(), e); + if (discoverer != null) { + try { + URL u = new URL(modelUrl); + discoverer.blacklist(u); + } catch (MalformedURLException e1) { + } + } + } + } + return null; + } + + @Override + public synchronized void initialize(Context context) { + + try { + Optional<ServiceDiscoverer> discovererOpt = (Optional) (context.getCapability(Context.Capabilities.SERVICE_DISCOVERER)); + if (discovererOpt.isPresent()) { + discoverer = discovererOpt.get(); + } + else { + Optional<Object> clientOptional = context.getCapability(Context.Capabilities.ZOOKEEPER_CLIENT); + CuratorFramework client = null; + if (clientOptional.isPresent() && clientOptional.get() instanceof CuratorFramework) { + client = (CuratorFramework) clientOptional.get(); + } else { + throw new IllegalStateException("Unable to initialize function: Cannot find zookeeper client."); + } + discoverer = createDiscoverer(client); + } + } + catch(Exception ex) { + LOG.error(ex.getMessage(), ex); + } + finally { + //We always want to set initialize to true because we don't want to keep trying to initialize over and over + isInitialized = true; + } + } + + @Override + public boolean isInitialized() { + return isInitialized; + } + } + + private static ServiceDiscoverer createDiscoverer(CuratorFramework client) throws Exception { + MaaSConfig config = ConfigUtil.INSTANCE.read(client, "/metron/maas/config", new MaaSConfig(), MaaSConfig.class); + ServiceDiscoverer discoverer = new ServiceDiscoverer(client, config.getServiceRoot()); + discoverer.start(); + return discoverer; + } + + @Stellar(name="GET_ENDPOINT" + , namespace="MAAS" + , description="Inspects ZooKeeper and returns a map containing the name, version and url for the model referred to by the input parameters." + , params = { + "model_name - The name of the model" + ,"model_version - The optional version of the model. If the model version is not specified, the most current version is used." + } + , returns = "A map containing the name, version, and url for the REST endpoint (fields named name, version and url). " + + "Note that the output of this function is suitable for input into the first argument of MAAS_MODEL_APPLY." + ) + public static class GetEndpoint implements StellarFunction { + ServiceDiscoverer discoverer; + private boolean isInitialized = false; + private boolean isValidState = false; + + @Override + public Object apply(List<Object> args, Context context) throws ParseException { + if(!isValidState) { + LOG.error("Invalid state: Unable to find ServiceDiscoverer service."); + return null; + } + String modelName = null; + String modelVersion = null; + if(args.size() >= 1) { + modelName = args.get(0).toString(); + } + if(args.size() >= 2) + { + modelVersion = args.get(1).toString(); + } + if(modelName == null) { + return null; + } + try { + ModelEndpoint ep = null; + if (modelVersion == null) { + ep = discoverer.getEndpoint(modelName); + } else { + ep = discoverer.getEndpoint(modelName, modelVersion); + } + return ep == null ? null : endpointToMap(ep.getName(), ep.getVersion(), ep.getEndpoint()); + } + catch(Exception ex) { + LOG.error("Unable to discover endpoint: {}", ex.getMessage(), ex); + return null; + } + } + + public static Map<String, String> endpointToMap(String name, String version, Endpoint ep) { + Map<String, String> ret = new HashMap<>(); + ret.put("url", ep.getUrl()); + ret.put("name", name); + ret.put("version", version); + for(Map.Entry<String, String> kv : ep.getFunctions().entrySet()) { + ret.put("endpoint:" + kv.getKey(), kv.getValue()); + } + return ret; + } + + @Override + public synchronized void initialize(Context context) { + try { + Optional<Object> clientOptional = context.getCapability(Context.Capabilities.ZOOKEEPER_CLIENT); + CuratorFramework client = null; + if (clientOptional.isPresent() && clientOptional.get() instanceof CuratorFramework) { + client = (CuratorFramework) clientOptional.get(); + } else { + throw new IllegalStateException("Unable to initialize function: Cannot find zookeeper client."); + } + try { + discoverer = createDiscoverer(client); + context.addCapability(Context.Capabilities.SERVICE_DISCOVERER, () -> discoverer); + isValidState = true; + } catch (Exception e) { + LOG.error(e.getMessage(), e); + throw new IllegalStateException("Unable to initialize MAAS_GET_ENDPOINT", e); + } + } + finally { + isInitialized = true; + } + } + + @Override + public boolean isInitialized() { + return isInitialized; + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-analytics/metron-maas-service/README.md ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-maas-service/README.md b/metron-analytics/metron-maas-service/README.md index 30f785d..575c873 100644 --- a/metron-analytics/metron-maas-service/README.md +++ b/metron-analytics/metron-maas-service/README.md @@ -126,16 +126,29 @@ Now let's install some prerequisites: Start Squid via `service squid start` Now that we have flask and jinja, we can create a mock DGA service to deploy with MaaS: -* Download the files in [this](https://gist.github.com/cestella/cba10aff0f970078a4c2c8cade3a4d1a) gist into the `/root/mock_dga` directory -* Make `rest.sh` executable via `chmod +x /root/mock_dga/rest.sh` +* Download the files in [this](https://gist.github.com/cestella/cba10aff0f970078a4c2c8cade3a4d1a) gist into the `$HOME/mock_dga` directory +* Make `rest.sh` executable via `chmod +x $HOME/mock_dga/rest.sh` This service will treat `yahoo.com` and `amazon.com` as legit and everything else as malicious. The contract is that the REST service exposes an endpoint `/apply` and returns back JSON maps with a single key `is_malicious` which can be `malicious` or `legit`. ## Deploy Mock DGA Service via MaaS +The following presumes that you are a logged in as a user who has a +home directory in HDFS under `/user/$USER`. If you do not, please create one +and ensure the permissions are set appropriate: +``` +su - hdfs -c "hadoop fs -mkdir /user/$USER" +su - hdfs -c "hadoop fs -chown $USER:$USER /user/$USER" +``` +Or, in the common case for the `metron` user: +``` +su - hdfs -c "hadoop fs -mkdir /user/metron" +su - hdfs -c "hadoop fs -chown metron:metron /user/metron" +``` + Now let's start MaaS and deploy the Mock DGA Service: * Start MaaS via `$METRON_HOME/bin/maas_service.sh -zq node1:2181` -* Start one instance of the mock DGA model with 512M of memory via `$METRON_HOME/bin/maas_deploy.sh -zq node1:2181 -lmp /root/mock_dga -hmp /user/root/models -mo ADD -m 512 -n dga -v 1.0 -ni 1` +* Start one instance of the mock DGA model with 512M of memory via `$METRON_HOME/bin/maas_deploy.sh -zq node1:2181 -lmp $HOME/mock_dga -hmp /user/$USER/models -mo ADD -m 512 -n dga -v 1.0 -ni 1` * As a sanity check: * Ensure that the model is running via `$METRON_HOME/bin/maas_deploy.sh -zq node1:2181 -mo LIST`. You should see `Model dga @ 1.0` be displayed and under that a url such as (but not exactly) `http://node1:36161` * Try to hit the model via curl: `curl 'http://localhost:36161/apply?host=caseystella.com'` and ensure that it returns a JSON map indicating the domain is malicious. @@ -170,8 +183,6 @@ Now that we have a deployed model, let's adjust the configurations for the Squid * Edit the squid enrichment configuration at `$METRON_HOME/config/zookeeper/enrichments/squid.json` (this file will not exist, so create a new one) to make the threat triage adjust the level of risk based on the model output: ``` { - "index": "squid", - "batchSize": 1, "enrichment" : { "fieldMap": {} }, http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-analytics/metron-maas-service/pom.xml ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-maas-service/pom.xml b/metron-analytics/metron-maas-service/pom.xml index 0ac9bac..4eeceae 100644 --- a/metron-analytics/metron-maas-service/pom.xml +++ b/metron-analytics/metron-maas-service/pom.xml @@ -49,17 +49,6 @@ <version>${global_kryo_serializers_version}</version> </dependency> <dependency> - <groupId>org.apache.metron</groupId> - <artifactId>stellar-common</artifactId> - <version>${project.parent.version}</version> - <exclusions> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-auth</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-server-common</artifactId> <version>${hadoop.version}</version> http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java deleted file mode 100644 index eacb64d..0000000 --- a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java +++ /dev/null @@ -1,324 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.maas.functions; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import java.lang.invoke.MethodHandles; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.TimeUnit; -import org.apache.curator.framework.CuratorFramework; -import org.apache.metron.maas.config.Endpoint; -import org.apache.metron.maas.config.MaaSConfig; -import org.apache.metron.maas.config.ModelEndpoint; -import org.apache.metron.maas.discovery.ServiceDiscoverer; -import org.apache.metron.maas.util.ConfigUtil; -import org.apache.metron.maas.util.RESTUtil; -import org.apache.metron.stellar.common.utils.JSONUtils; -import org.apache.metron.stellar.dsl.Context; -import org.apache.metron.stellar.dsl.ParseException; -import org.apache.metron.stellar.dsl.Stellar; -import org.apache.metron.stellar.dsl.StellarFunction; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class MaaSFunctions { - protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private static class ModelCacheKey { - String name; - String version; - String method; - Map<String, String> args; - public ModelCacheKey(String name, String version, String method, Map<String, String> args) { - this.name = name; - this.version = version; - this.method = method; - this.args = args; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - ModelCacheKey that = (ModelCacheKey) o; - - if (name != null ? !name.equals(that.name) : that.name != null) return false; - if (version != null ? !version.equals(that.version) : that.version != null) return false; - if (method != null ? !method.equals(that.method) : that.method != null) return false; - return args != null ? args.equals(that.args) : that.args == null; - - } - - @Override - public int hashCode() { - int result = name != null ? name.hashCode() : 0; - result = 31 * result + (version != null ? version.hashCode() : 0); - result = 31 * result + (method != null ? method.hashCode() : 0); - result = 31 * result + (args != null ? args.hashCode() : 0); - return result; - } - } - - @Stellar(name="MODEL_APPLY" - , namespace="MAAS" - , description = "Returns the output of a model deployed via Model as a Service. NOTE: Results are cached locally for 10 minutes." - , params = { "endpoint - A map containing the name, version, and url for the REST endpoint" - , "function - The optional endpoint path; default is 'apply'" - , "model_args - A Dictionary of arguments for the model (these become request params)" - } - , returns = "The output of the model deployed as a REST endpoint in Map form. Assumes REST endpoint returns a JSON Map." - ) - public static class ModelApply implements StellarFunction { - private boolean isInitialized = false; - private ServiceDiscoverer discoverer; - private Cache<ModelCacheKey, Map<String, Object> > resultCache; - public ModelApply() { - resultCache = CacheBuilder.newBuilder() - .concurrencyLevel(4) - .weakKeys() - .maximumSize(100000) - .expireAfterWrite(10, TimeUnit.MINUTES) - .build(); - } - - @Override - public Object apply(List<Object> args, Context context) throws ParseException { - if(args.size() < 2) { - throw new ParseException("Unable to execute model_apply. " + - "Expected arguments: endpoint_map:map, " + - " [endpoint method:string], model_args:map" - ); - } - if(!isInitialized) { - return null; - } - int i = 0; - if(args.size() == 0) { - return null; - } - Object endpointObj = args.get(i++); - Map endpoint = null; - String modelName; - String modelVersion; - String modelUrl; - if(endpointObj instanceof Map) { - endpoint = (Map)endpointObj; - modelName = endpoint.get("name") + ""; - modelVersion = endpoint.get("version") + ""; - modelUrl = endpoint.get("url") + ""; - } - else { - return null; - } - String modelFunction = "apply"; - Map<String, String> modelArgs = new HashMap<>(); - if(args.get(i) instanceof String) { - String func = (String)args.get(i); - if(endpoint.containsKey("endpoint:" + func)) { - modelFunction = "" + endpoint.get("endpoint:" + func); - } - else { - modelFunction = func; - } - i++; - } - - if(args.get(i) instanceof Map) { - if(endpoint.containsKey("endpoint:apply")) { - modelFunction = "" + endpoint.get("endpoint:apply"); - } - modelArgs = (Map)args.get(i); - } - if( modelName == null - || modelVersion == null - || modelFunction == null - ) { - return null; - } - ModelCacheKey cacheKey = new ModelCacheKey(modelName, modelVersion, modelFunction, modelArgs); - Map<String, Object> ret = resultCache.getIfPresent(cacheKey); - if(ret != null) { - return ret; - } - else { - String url = modelUrl; - if (url.endsWith("/")) { - url = url.substring(0, url.length() - 1); - } - if (modelFunction.startsWith("/")) { - modelFunction = modelFunction.substring(1); - } - try { - URL u = new URL(url + "/" + modelFunction); - - String results = RESTUtil.INSTANCE.getRESTJSONResults(u, modelArgs); - ret = JSONUtils.INSTANCE.load(results, new TypeReference<Map<String, Object>>() { - }); - resultCache.put(cacheKey, ret); - return ret; - } catch (Exception e) { - LOG.error(e.getMessage(), e); - if (discoverer != null) { - try { - URL u = new URL(modelUrl); - discoverer.blacklist(u); - } catch (MalformedURLException e1) { - } - } - } - } - return null; - } - - @Override - public synchronized void initialize(Context context) { - - try { - Optional<ServiceDiscoverer> discovererOpt = (Optional) (context.getCapability(Context.Capabilities.SERVICE_DISCOVERER)); - if (discovererOpt.isPresent()) { - discoverer = discovererOpt.get(); - } - else { - Optional<Object> clientOptional = context.getCapability(Context.Capabilities.ZOOKEEPER_CLIENT); - CuratorFramework client = null; - if (clientOptional.isPresent() && clientOptional.get() instanceof CuratorFramework) { - client = (CuratorFramework) clientOptional.get(); - } else { - throw new IllegalStateException("Unable to initialize function: Cannot find zookeeper client."); - } - discoverer = createDiscoverer(client); - } - } - catch(Exception ex) { - LOG.error(ex.getMessage(), ex); - } - finally { - //We always want to set initialize to true because we don't want to keep trying to initialize over and over - isInitialized = true; - } - } - - @Override - public boolean isInitialized() { - return isInitialized; - } - } - - private static ServiceDiscoverer createDiscoverer(CuratorFramework client) throws Exception { - MaaSConfig config = ConfigUtil.INSTANCE.read(client, "/metron/maas/config", new MaaSConfig(), MaaSConfig.class); - ServiceDiscoverer discoverer = new ServiceDiscoverer(client, config.getServiceRoot()); - discoverer.start(); - return discoverer; - } - - @Stellar(name="GET_ENDPOINT" - , namespace="MAAS" - , description="Inspects ZooKeeper and returns a map containing the name, version and url for the model referred to by the input parameters." - , params = { - "model_name - The name of the model" - ,"model_version - The optional version of the model. If the model version is not specified, the most current version is used." - } - , returns = "A map containing the name, version, and url for the REST endpoint (fields named name, version and url). " + - "Note that the output of this function is suitable for input into the first argument of MAAS_MODEL_APPLY." - ) - public static class GetEndpoint implements StellarFunction { - ServiceDiscoverer discoverer; - private boolean isInitialized = false; - private boolean isValidState = false; - - @Override - public Object apply(List<Object> args, Context context) throws ParseException { - if(!isValidState) { - LOG.error("Invalid state: Unable to find ServiceDiscoverer service."); - return null; - } - String modelName = null; - String modelVersion = null; - if(args.size() >= 1) { - modelName = args.get(0).toString(); - } - if(args.size() >= 2) - { - modelVersion = args.get(1).toString(); - } - if(modelName == null) { - return null; - } - try { - ModelEndpoint ep = null; - if (modelVersion == null) { - ep = discoverer.getEndpoint(modelName); - } else { - ep = discoverer.getEndpoint(modelName, modelVersion); - } - return ep == null ? null : endpointToMap(ep.getName(), ep.getVersion(), ep.getEndpoint()); - } - catch(Exception ex) { - LOG.error("Unable to discover endpoint: {}", ex.getMessage(), ex); - return null; - } - } - - public static Map<String, String> endpointToMap(String name, String version, Endpoint ep) { - Map<String, String> ret = new HashMap<>(); - ret.put("url", ep.getUrl()); - ret.put("name", name); - ret.put("version", version); - for(Map.Entry<String, String> kv : ep.getFunctions().entrySet()) { - ret.put("endpoint:" + kv.getKey(), kv.getValue()); - } - return ret; - } - - @Override - public synchronized void initialize(Context context) { - try { - Optional<Object> clientOptional = context.getCapability(Context.Capabilities.ZOOKEEPER_CLIENT); - CuratorFramework client = null; - if (clientOptional.isPresent() && clientOptional.get() instanceof CuratorFramework) { - client = (CuratorFramework) clientOptional.get(); - } else { - throw new IllegalStateException("Unable to initialize function: Cannot find zookeeper client."); - } - try { - discoverer = createDiscoverer(client); - context.addCapability(Context.Capabilities.SERVICE_DISCOVERER, () -> discoverer); - isValidState = true; - } catch (Exception e) { - LOG.error(e.getMessage(), e); - throw new IllegalStateException("Unable to initialize MAAS_GET_ENDPOINT", e); - } - } - finally { - isInitialized = true; - } - } - - @Override - public boolean isInitialized() { - return isInitialized; - } - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/Client.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/Client.java b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/Client.java index c2d8906..9cabf21 100644 --- a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/Client.java +++ b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/Client.java @@ -24,6 +24,7 @@ import java.util.*; import java.util.function.Function; import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import org.apache.commons.cli.*; import org.apache.commons.cli.CommandLine; @@ -558,6 +559,7 @@ public class Client { // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path FileSystem fs = FileSystem.get(conf); + createMaaSDirectory(fs, appId.toString()); Path ajPath = addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.toString(), localResources, null); // Set the log4j properties if needed @@ -789,6 +791,18 @@ public class Client { yarnClient.killApplication(appId); } + private void createMaaSDirectory(FileSystem fs, String appId) throws IOException { + for(Path p : ImmutableList.of(new Path(fs.getHomeDirectory(), appName) + , new Path(fs.getHomeDirectory(), appName + "/" + appId) + ) + ) { + if(!fs.exists(p)) { + fs.mkdirs(p); + fs.setPermission(p, new FsPermission((short)0755)); + } + } + } + private Path addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId, Map<String, LocalResource> localResources, String resources) throws IOException { @@ -808,6 +822,7 @@ public class Client { } else { fs.copyFromLocalFile(new Path(fileSrcPath), dst); } + fs.setPermission(dst, new FsPermission((short)0755)); FileStatus scFileStatus = fs.getFileStatus(dst); LocalResource scRsrc = LocalResource.newInstance( http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/Constants.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/Constants.java b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/Constants.java index ac2c950..d032511 100644 --- a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/Constants.java +++ b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/Constants.java @@ -31,5 +31,8 @@ public class Constants { * Environment key name denoting the timeline domain ID. */ public static final String TIMELINEDOMAIN = "TIMELINEDOMAIN"; - + /* + The filename which communicates the endpoint information for a deployed model + */ + public static final String ENDPOINT_DAT = "endpoint.dat"; } http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java index cc297d2..8f0b9e5 100644 --- a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java +++ b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java @@ -33,6 +33,7 @@ import org.apache.curator.utils.CloseableUtils; import org.apache.curator.x.discovery.*; import org.apache.curator.x.discovery.details.JsonInstanceSerializer; import org.apache.metron.maas.config.Endpoint; +import org.apache.metron.maas.service.Constants; import org.apache.metron.maas.util.ConfigUtil; import org.apache.metron.maas.config.MaaSConfig; import org.apache.metron.maas.config.ModelEndpoint; @@ -202,7 +203,11 @@ public class Runner { serviceDiscovery.start(); File cwd = new File(script).getParentFile(); - final String cmd = new File(cwd, script).getAbsolutePath(); + File scriptFile = new File(cwd, script); + if(scriptFile.exists() && !scriptFile.canExecute()) { + scriptFile.setExecutable(true); + } + final String cmd = scriptFile.getAbsolutePath(); try { p = new ProcessBuilder(cmd).directory(cwd).start(); @@ -299,7 +304,7 @@ public class Runner { private static Endpoint readEndpoint(File cwd) throws Exception { String content = ""; - File f = new File(cwd, "endpoint.dat"); + File f = new File(cwd, Constants.ENDPOINT_DAT); for(int i = 0;i < NUM_ATTEMPTS;i++) { if(f.exists()) { try { http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/submit/ModelSubmission.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/submit/ModelSubmission.java b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/submit/ModelSubmission.java index ebfa904..fcae40a 100644 --- a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/submit/ModelSubmission.java +++ b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/submit/ModelSubmission.java @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.Path; import org.apache.log4j.PropertyConfigurator; import org.apache.metron.maas.config.*; import org.apache.metron.maas.discovery.ServiceDiscoverer; +import org.apache.metron.maas.service.Constants; import org.apache.metron.maas.service.Log4jPropertyHelper; import org.apache.metron.maas.util.ConfigUtil; import org.apache.metron.maas.queue.Queue; @@ -247,6 +248,10 @@ public class ModelSubmission { fs.mkdirs(hdfsPath); } for(File f : localDir.listFiles()) { + if(f.getName().equals(Constants.ENDPOINT_DAT)) { + //skip the endpoint if it exists accidentally, we don't want to localize that. + continue; + } Path p = new Path(hdfsPath, f.getName()); FSDataOutputStream out = fs.create(p); BufferedInputStream in = new BufferedInputStream(new FileInputStream(f)); http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json index 820179a..015f026 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json @@ -5,11 +5,13 @@ "_comment" : "dependencies for all cases", "METRON_INDEXING-INSTALL" : ["METRON_PARSERS-INSTALL"], "METRON_ENRICHMENT-INSTALL": ["METRON_INDEXING-INSTALL"], + "METRON_PROFILER-INSTALL": ["METRON_ENRICHMENT-INSTALL"], "METRON_REST-INSTALL": ["METRON_PARSERS-INSTALL"], "METRON_PARSERS-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START","METRON_ENRICHMENT_MASTER-START"], "METRON_ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"], "METRON_ENRICHMENT_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_ENRICHMENT_MASTER-START"], "METRON_INDEXING-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START","METRON_PARSERS-START"], + "METRON_PROFILER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START", "METRON_ENRICHMENT-INSTALL"], "METRON_REST-START": ["KAFKA_BROKER-START","STORM_REST_API-START","ZOOKEEPER_SERVER-START","NAMENODE-START","METRON_PARSERS-INSTALL","METRON_INDEXING-INSTALL","METRON_ENRICHMENT-INSTALL"], "METRON_MANAGEMENT_UI-START": ["METRON_REST-START"], "STORM_REST_API-STOP" : ["METRON_ENRICHMENT_MASTER-STOP","METRON_PARSERS-STOP","METRON_INDEXING-STOP","METRON_REST-STOP","METRON_MANAGEMENT_UI-STOP"], http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml index e36730a..6abbe77 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml @@ -59,7 +59,7 @@ <description>Indexing Writer Class Name</description> <value>org.apache.metron.elasticsearch.writer.ElasticsearchWriter</value> <display-name>Indexing Writer Class Name</display-name> - </property> + </property> <property> <name>update_table</name> <description>The HBase table which will hold edits to indexed data</description> http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml new file mode 100644 index 0000000..c7f6ce2 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml @@ -0,0 +1,155 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration supports_final="true"> + <property> + <name>profiler_kafka_start</name> + <value>UNCOMMITTED_EARLIEST</value> + <description>One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST</description> + <display-name>Input Topic Start</display-name> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>EARLIEST</value> + </entry> + <entry> + <value>LATEST</value> + </entry> + <entry> + <value>UNCOMMITTED_EARLIEST</value> + </entry> + <entry> + <value>UNCOMMITTED_LATEST</value> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + </property> + <property> + <name>profiler_period_duration</name> + <value>15</value> + <description>The duration of each profile period. This value should be defined along with profiler.period.duration.units</description> + <display-name>Period Duration</display-name> + </property> + <property> + <name>profiler_period_units</name> + <value>MINUTES</value> + <description>The units used to specify the profiler.period.duration. This value should be defined along with profiler.period.duration.</description> + <display-name>Period Units</display-name> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>DAYS</value> + </entry> + <entry> + <value>HOURS</value> + </entry> + <entry> + <value>MINUTES</value> + </entry> + <entry> + <value>SECONDS</value> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + + </property> + <property> + <name>profiler_ttl</name> + <value>30</value> + <description>If a message has not been applied to a Profile in this period of time, the Profile will be terminated and its resources will be cleaned up. This value should be defined along with profiler.ttl.units. + This time-to-live does not affect the persisted Profile data in HBase. It only affects the state stored in memory during the execution of the latest profile period. This state will be deleted if the time-to-live is exceeded. + </description> + <display-name>Time to Live</display-name> + </property> + <property> + <name>profiler_ttl_units</name> + <value>MINUTES</value> + <description>The units used to specify the profiler.ttl.</description> + <display-name>Time To Live Units</display-name> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>DAYS</value> + </entry> + <entry> + <value>HOURS</value> + </entry> + <entry> + <value>MINUTES</value> + </entry> + <entry> + <value>SECONDS</value> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + + + </property> + <property> + <name>profiler_hbase_table</name> + <value>profiler</value> + <description>The name of the HBase table that profile data is written to. The Profiler expects that the table exists and is writable.</description> + <display-name>HBase Table</display-name> + </property> + <property> + <name>profiler_hbase_cf</name> + <value>P</value> + <description>The column family used to store profile data in HBase.</description> + <display-name>HBase Table Column Family</display-name> + </property> + <property> + <name>profiler_hbase_batch</name> + <value>10</value> + <description>The number of puts that are written to HBase in a single batch.</description> + <display-name>HBase Batch Size</display-name> + </property> + <property> + <name>profiler_hbase_flush_interval</name> + <value>30</value> + <description>The maximum number of seconds between batch writes to HBase.</description> + <display-name>HBase Flush Interval</display-name> + </property> + <property> + <name>profiler_topology_worker_childopts</name> + <value/> + <description>Extra topology child opts for the storm topology.</description> + <display-name>topology.worker.childopts</display-name> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + <property> + <name>profiler_topology_workers</name> + <value>1</value> + <description>The profiler storm topology workers</description> + <display-name>Number of Workers</display-name> + </property> + <property> + <name>profiler_acker_executors</name> + <value>1</value> + <description>The profiler storm topology acker executors</description> + <display-name>Number of Acker Executors</display-name> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml index 77f247c..2844605 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml @@ -119,6 +119,46 @@ </component> <component> + <name>METRON_PROFILER</name> + <displayName>Metron Profiler</displayName> + <category>MASTER</category> + <cardinality>1</cardinality> + <versionAdvertised>false</versionAdvertised> + <dependencies> + <dependency> + <name>HBASE/HBASE_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>ZOOKEEPER/ZOOKEEPER_SERVER</name> + <scope>cluster</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>KAFKA/KAFKA_BROKER</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> + <commandScript> + <script>scripts/profiler_master.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + <configuration-dependencies> + <config-type>metron-enrichment-env</config-type> + <config-type>metron-profiler-env</config-type> + </configuration-dependencies> + </component> + + <component> <name>METRON_INDEXING</name> <displayName>Metron Indexing</displayName> <category>MASTER</category> @@ -321,6 +361,9 @@ <name>metron-enrichment</name> </package> <package> + <name>metron-profiler</name> + </package> + <package> <name>metron-indexing</name> </package> <package> http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index e9426db..abbddc5 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -180,6 +180,7 @@ enrichment_cf = status_params.enrichment_cf update_table = status_params.update_table update_cf = status_params.update_cf + threatintel_table = status_params.threatintel_table threatintel_cf = status_params.threatintel_cf @@ -210,6 +211,7 @@ security_enabled = status_params.security_enabled client_jaas_path = metron_home + '/client_jaas.conf' client_jaas_arg = '-Djava.security.auth.login.config=' + metron_home + '/client_jaas.conf' enrichment_topology_worker_childopts = client_jaas_arg if security_enabled else '' +profiler_topology_worker_childopts = client_jaas_arg if security_enabled else '' indexing_topology_worker_childopts = client_jaas_arg if security_enabled else '' metron_jvm_flags += (' ' + client_jaas_arg) if security_enabled else '' topology_auto_credentials = config['configurations']['storm-site'].get('nimbus.credential.renewers.classes', []) @@ -260,6 +262,29 @@ threat_intel_stellar_parallelism = config['configurations']['metron-enrichment-e threat_intel_join_parallelism = config['configurations']['metron-enrichment-env']['threat_intel_join_parallelism'] kafka_writer_parallelism = config['configurations']['metron-enrichment-env']['kafka_writer_parallelism'] +# Profiler + +metron_profiler_topology = 'profiler' +profiler_input_topic = config['configurations']['metron-enrichment-env']['enrichment_output_topic'] +profiler_kafka_start = config['configurations']['metron-profiler-env']['profiler_kafka_start'] +profiler_period_duration = config['configurations']['metron-profiler-env']['profiler_period_duration'] +profiler_period_units = config['configurations']['metron-profiler-env']['profiler_period_units'] +profiler_ttl = config['configurations']['metron-profiler-env']['profiler_ttl'] +profiler_ttl_units = config['configurations']['metron-profiler-env']['profiler_ttl_units'] +profiler_hbase_batch = config['configurations']['metron-profiler-env']['profiler_hbase_batch'] +profiler_hbase_flush_interval = config['configurations']['metron-profiler-env']['profiler_hbase_flush_interval'] +profiler_topology_workers = config['configurations']['metron-profiler-env']['profiler_topology_workers'] +profiler_acker_executors = config['configurations']['metron-profiler-env']['profiler_acker_executors'] +profiler_hbase_table = config['configurations']['metron-profiler-env']['profiler_hbase_table'] +profiler_hbase_cf = config['configurations']['metron-profiler-env']['profiler_hbase_cf'] +profiler_configured_flag_file = status_params.profiler_configured_flag_file +profiler_acl_configured_flag_file = status_params.indexing_acl_configured_flag_file +profiler_hbase_configured_flag_file = status_params.profiler_hbase_configured_flag_file +profiler_hbase_acl_configured_flag_file = status_params.profiler_hbase_acl_configured_flag_file +if not len(profiler_topology_worker_childopts) == 0: + profiler_topology_worker_childopts += ' ' +profiler_topology_worker_childopts += config['configurations']['metron-profiler-env']['profiler_topology_worker_childopts'] + # Indexing indexing_kafka_start = config['configurations']['metron-indexing-env']['indexing_kafka_start'] indexing_input_topic = status_params.indexing_input_topic http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py index 215d808..76f8570 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py @@ -61,6 +61,17 @@ threatintel_cf = 't' update_table = 'metron_update' update_cf = 't' +# Profiler +metron_profiler_topology = 'profiler' +profiler_input_topic = config['configurations']['metron-enrichment-env']['enrichment_output_topic'] +profiler_hbase_table = config['configurations']['metron-profiler-env']['profiler_hbase_table'] +profiler_hbase_cf = config['configurations']['metron-profiler-env']['profiler_hbase_cf'] +profiler_configured_flag_file = metron_zookeeper_config_path + '/../metron_profiler_configured' +profiler_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_profiler_acl_configured' +profiler_hbase_configured_flag_file = metron_zookeeper_config_path + '/../metron_profiler_hbase_configured' +profiler_hbase_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_profiler_hbase_acl_configured' + + # Indexing metron_indexing_topology = 'indexing' indexing_input_topic = config['configurations']['metron-indexing-env']['indexing_input_topic'] http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py new file mode 100644 index 0000000..ddd66cb --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import time + +from resource_management.core.logger import Logger +from resource_management.core.resources.system import Execute, File + +import metron_service +import metron_security + + +# Wrap major operations and functionality in this class +class ProfilerCommands: + __params = None + __profiler_topic = None + __profiler_topology = None + __configured = False + __acl_configured = False + __hbase_configured = False + __hbase_acl_configured = False + + def __init__(self, params): + if params is None: + raise ValueError("params argument is required for initialization") + self.__params = params + self.__profiler_topology = params.metron_profiler_topology + self.__profiler_topic = params.profiler_input_topic + self.__configured = os.path.isfile(self.__params.profiler_configured_flag_file) + self.__acl_configured = os.path.isfile(self.__params.profiler_acl_configured_flag_file) + self.__hbase_configured = os.path.isfile(self.__params.profiler_hbase_configured_flag_file) + self.__hbase_acl_configured = os.path.isfile(self.__params.profiler_hbase_acl_configured_flag_file) + + def is_configured(self): + return self.__configured + + def is_acl_configured(self): + return self.__acl_configured + + def set_configured(self): + File(self.__params.profiler_configured_flag_file, + content="", + owner=self.__params.metron_user, + mode=0755) + + def is_hbase_configured(self): + return self.__hbase_configured + + def is_hbase_acl_configured(self): + return self.__hbase_acl_configured + + def set_hbase_configured(self): + Logger.info("Setting HBase Configured to True") + File(self.__params.profiler_hbase_configured_flag_file, + content="", + owner=self.__params.metron_user, + mode=0755) + + def set_hbase_acl_configured(self): + Logger.info("Setting HBase ACL Configured to True") + File(self.__params.profiler_hbase_acl_configured_flag_file, + content="", + owner=self.__params.metron_user, + mode=0755) + + def create_hbase_tables(self): + Logger.info("Creating HBase Tables") + if self.__params.security_enabled: + metron_security.kinit(self.__params.kinit_path_local, + self.__params.hbase_keytab_path, + self.__params.hbase_principal_name, + execute_user=self.__params.hbase_user) + cmd = "echo \"create '{0}','{1}'\" | hbase shell -n" + add_table_cmd = cmd.format(self.__params.profiler_hbase_table, self.__params.profiler_hbase_cf) + Execute(add_table_cmd, + tries=3, + try_sleep=5, + logoutput=False, + path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', + user=self.__params.hbase_user + ) + + Logger.info("Done creating HBase Tables") + self.set_hbase_configured() + + def set_hbase_acls(self): + Logger.info("Setting HBase ACLs") + if self.__params.security_enabled: + metron_security.kinit(self.__params.kinit_path_local, + self.__params.hbase_keytab_path, + self.__params.hbase_principal_name, + execute_user=self.__params.hbase_user) + cmd = "echo \"grant '{0}', 'RW', '{1}'\" | hbase shell -n" + add_table_acl_cmd = cmd.format(self.__params.metron_user, self.__params.profiler_hbase_table) + Execute(add_table_acl_cmd, + tries=3, + try_sleep=5, + logoutput=False, + path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', + user=self.__params.hbase_user + ) + + Logger.info("Done setting HBase ACLs") + self.set_hbase_acl_configured() + + def set_acl_configured(self): + File(self.__params.profiler_acl_configured_flag_file, + content="", + owner=self.__params.metron_user, + mode=0755) + + def start_profiler_topology(self, env): + Logger.info('Starting ' + self.__profiler_topology) + + if not self.is_topology_active(env): + if self.__params.security_enabled: + metron_security.kinit(self.__params.kinit_path_local, + self.__params.metron_keytab_path, + self.__params.metron_principal_name, + execute_user=self.__params.metron_user) + start_cmd_template = """{0}/bin/start_profiler_topology.sh \ + -s {1} \ + -z {2}""" + Execute(start_cmd_template.format(self.__params.metron_home, + self.__profiler_topology, + self.__params.zookeeper_quorum), + user=self.__params.metron_user) + + else: + Logger.info('Profiler topology already running') + + Logger.info('Finished starting profiler topology') + + def stop_profiler_topology(self, env): + Logger.info('Stopping ' + self.__profiler_topology) + + if self.is_topology_active(env): + if self.__params.security_enabled: + metron_security.kinit(self.__params.kinit_path_local, + self.__params.metron_keytab_path, + self.__params.metron_principal_name, + execute_user=self.__params.metron_user) + stop_cmd = 'storm kill ' + self.__profiler_topology + Execute(stop_cmd, user=self.__params.metron_user) + + else: + Logger.info("Profiler topology already stopped") + + Logger.info('Done stopping profiler topologies') + + def restart_profiler_topology(self, env): + Logger.info('Restarting the profiler topologies') + self.stop_profiler_topology(env) + + # Wait for old topology to be cleaned up by Storm, before starting again. + retries = 0 + topology_active = self.is_topology_active(env) + while self.is_topology_active(env) and retries < 3: + Logger.info('Existing topology still active. Will wait and retry') + time.sleep(10) + retries += 1 + + if not topology_active: + Logger.info('Waiting for storm kill to complete') + time.sleep(30) + self.start_profiler_topology(env) + Logger.info('Done restarting the profiler topologies') + else: + Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.') + + def is_topology_active(self, env): + env.set_params(self.__params) + active = True + topologies = metron_service.get_running_topologies(self.__params) + is_running = False + if self.__profiler_topology in topologies: + is_running = topologies[self.__profiler_topology] in ['ACTIVE', 'REBALANCING'] + active &= is_running + return active http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py new file mode 100644 index 0000000..4946ab0 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py @@ -0,0 +1,94 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +from resource_management.core.exceptions import ComponentIsNotRunning +from resource_management.core.logger import Logger +from resource_management.core.resources.system import Execute +from resource_management.core.resources.system import File +from resource_management.core.source import Template +from resource_management.libraries.functions.format import format +from resource_management.core.source import StaticFile +from resource_management.libraries.functions import format as ambari_format +from resource_management.libraries.script import Script + +from metron_security import storm_security_setup +import metron_service +from profiler_commands import ProfilerCommands + + +class Profiler(Script): + __configured = False + + def install(self, env): + from params import params + env.set_params(params) + self.install_packages(env) + + def configure(self, env, upgrade_type=None, config_dir=None): + from params import params + env.set_params(params) + + Logger.info("Running profiler configure") + File(format("{metron_config_path}/profiler.properties"), + content=Template("profiler.properties.j2"), + owner=params.metron_user, + group=params.metron_group + ) + + commands = ProfilerCommands(params) + metron_service.load_global_config(params) + + if not commands.is_configured(): + commands.set_configured() + + if not commands.is_hbase_configured(): + commands.create_hbase_tables() + if params.security_enabled and not commands.is_hbase_acl_configured(): + commands.set_hbase_acls() + + Logger.info("Calling security setup") + storm_security_setup(params) + + def start(self, env, upgrade_type=None): + from params import params + env.set_params(params) + self.configure(env) + commands = ProfilerCommands(params) + commands.start_profiler_topology(env) + + def stop(self, env, upgrade_type=None): + from params import params + env.set_params(params) + commands = ProfilerCommands(params) + commands.stop_profiler_topology(env) + + def status(self, env): + from params import status_params + env.set_params(status_params) + commands = ProfilerCommands(status_params) + if not commands.is_topology_active(env): + raise ComponentIsNotRunning() + + def restart(self, env): + from params import params + env.set_params(params) + self.configure(env) + commands = ProfilerCommands(params) + commands.restart_profiler_topology(env) + +if __name__ == "__main__": + Profiler().execute() http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/profiler.properties.j2 ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/profiler.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/profiler.properties.j2 new file mode 100644 index 0000000..cf2ad58 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/profiler.properties.j2 @@ -0,0 +1,47 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +##### Storm ##### + +topology.worker.childopts={{profiler_topology_worker_childopts}} + +##### Profiler ##### + +profiler.input.topic={{enrichment_output_topic}} +profiler.output.topic={{enrichment_input_topic}} +profiler.period.duration={{profiler_period_duration}} +profiler.period.duration.units={{profiler_period_units}} +profiler.workers={{profiler_topology_workers}} +profiler.executors={{profiler_acker_executors}} +profiler.ttl={{profiler_ttl}} +profiler.ttl.units={{profiler_ttl_units}} +profiler.hbase.salt.divisor=1000 +profiler.hbase.table={{profiler_hbase_table}} +profiler.hbase.column.family={{profiler_hbase_cf}} +profiler.hbase.batch={{profiler_hbase_batch}} +profiler.hbase.flush.interval.seconds={{profiler_hbase_flush_interval}} + +##### Kafka ##### + +kafka.zk={{zookeeper_quorum}} +kafka.broker={{kafka_brokers}} +# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST +kafka.start={{profiler_kafka_start}} +kafka.security.protocol={{kafka_security_protocol}} http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py index 4a95e63..2fb1ab0 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py @@ -43,6 +43,7 @@ class METRON${metron.short.version}ServiceAdvisor(service_advisor.ServiceAdvisor metronParsersHost = self.getHosts(componentsList, "METRON_PARSERS")[0] metronEnrichmentMaster = self.getHosts(componentsList, "METRON_ENRICHMENT_MASTER")[0] + metronProfilerMaster = self.getHosts(componentsList, "METRON_PROFILER")[0] metronIndexingHost = self.getHosts(componentsList, "METRON_INDEXING")[0] metronRESTHost = self.getHosts(componentsList, "METRON_REST")[0] @@ -76,6 +77,10 @@ class METRON${metron.short.version}ServiceAdvisor(service_advisor.ServiceAdvisor message = "Metron Indexing must be co-located with Metron Parsers on {0}".format(metronParsersHost) items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'METRON_INDEXING', "host": metronIndexingHost }) + if metronParsersHost != metronProfilerHost: + message = "Metron Profiler must be co-located with Metron Parsers on {0}".format(metronParsersHost) + items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'METRON_PROFILER', "host": metronProfilerHost }) + # Enrichment Master also needs ZK Client, but this is already guaranteed by being colocated with Parsers Master if metronParsersHost not in zookeeperClientHosts: message = "Metron must be co-located with an instance of Zookeeper Client" http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json index ce352a4..748feb8 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json @@ -65,7 +65,7 @@ "display-name": "Enrichment", "layout": { "tab-columns": "1", - "tab-rows": "4", + "tab-rows": "3", "sections": [ { "name": "section-enrichment-adapters", @@ -132,7 +132,7 @@ "display-name": "Indexing", "layout": { "tab-columns": "1", - "tab-rows": "3", + "tab-rows": "4", "sections": [ { "name": "section-indexing-kafka", @@ -155,7 +155,7 @@ }, { "name": "section-indexing-update", - "row-index": "0", + "row-index": "1", "column-index": "0", "row-span": "1", "column-span": "1", @@ -174,7 +174,7 @@ }, { "name": "section-indexing-storm", - "row-index": "1", + "row-index": "2", "column-index": "0", "row-span": "1", "column-span": "1", @@ -193,7 +193,7 @@ }, { "name": "section-indexing-hdfs", - "row-index": "2", + "row-index": "3", "column-index": "0", "row-span": "1", "column-span": "1", @@ -214,6 +214,73 @@ } }, { + "name": "profiler", + "display-name": "Profiler", + "layout": { + "tab-columns": "1", + "tab-rows": "3", + "sections": [ + { + "name": "section-profiler-kafka", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-profiler-kafka", + "display-name": "Kafka", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + }, + { + "name": "section-profiler-setup", + "row-index": "1", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-profiler-setup", + "display-name": "Profiler Setup", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + }, + { + "name": "section-profiler-storm", + "row-index": "2", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-profiler-storm", + "display-name": "Storm", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + } + ] + } + }, + { "name": "rest", "display-name": "REST", "layout": { @@ -454,6 +521,54 @@ }, { + "config": "metron-profiler-env/profiler_kafka_start", + "subsection-name": "subsection-profiler-kafka" + }, + { + "config": "metron-profiler-env/profiler_period_duration", + "subsection-name": "subsection-profiler-setup" + }, + { + "config": "metron-profiler-env/profiler_period_units", + "subsection-name": "subsection-profiler-setup" + }, + { + "config": "metron-profiler-env/profiler_ttl", + "subsection-name": "subsection-profiler-setup" + }, + { + "config": "metron-profiler-env/profiler_ttl_units", + "subsection-name": "subsection-profiler-setup" + }, + { + "config": "metron-profiler-env/profiler_hbase_table", + "subsection-name": "subsection-profiler-setup" + }, + { + "config": "metron-profiler-env/profiler_hbase_cf", + "subsection-name": "subsection-profiler-setup" + }, + { + "config": "metron-profiler-env/profiler_hbase_batch", + "subsection-name": "subsection-profiler-setup" + }, + { + "config": "metron-profiler-env/profiler_hbase_flush_interval", + "subsection-name": "subsection-profiler-setup" + }, + { + "config": "metron-profiler-env/profiler_topology_worker_childopts", + "subsection-name": "subsection-profiler-storm" + }, + { + "config": "metron-profiler-env/profiler_topology_workers", + "subsection-name": "subsection-profiler-storm" + }, + { + "config": "metron-profiler-env/profiler_acker_executors", + "subsection-name": "subsection-profiler-storm" + }, + { "config": "metron-rest-env/metron_rest_port", "subsection-name": "subsection-rest" }, @@ -780,7 +895,78 @@ "type": "text-field" } }, - + { + "config": "metron-profiler-env/profiler_kafka_start", + "widget": { + "type": "combo" + } + }, + { + "config": "metron-profiler-env/profiler_period_duration", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-profiler-env/profiler_period_units", + "widget": { + "type": "combo" + } + }, + { + "config": "metron-profiler-env/profiler_ttl", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-profiler-env/profiler_ttl_units", + "widget": { + "type": "combo" + } + }, + { + "config": "metron-profiler-env/profiler_hbase_table", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-profiler-env/profiler_hbase_cf", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-profiler-env/profiler_hbase_batch", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-profiler-env/profiler_hbase_flush_interval", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-profiler-env/profiler_topology_worker_childopts", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-profiler-env/profiler_topology_workers", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-profiler-env/profiler_acker_executors", + "widget": { + "type": "text-field" + } + }, { "config": "metron-rest-env/metron_rest_port",
