METRON-1587 Make collection utility work for HDP search (merrimanr) closes apache/metron#1043
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f241f87f Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f241f87f Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f241f87f Branch: refs/heads/master Commit: f241f87f87523f2c6f2203b3f316cff0fd3f20b4 Parents: aed4ffc Author: merrimanr <merrim...@gmail.com> Authored: Mon Jun 11 10:09:16 2018 -0500 Committer: merrimanr <merrim...@gmail.com> Committed: Mon Jun 11 10:09:16 2018 -0500 ---------------------------------------------------------------------- .../package/scripts/indexing_commands.py | 63 +++++++++++++++++--- .../CURRENT/package/scripts/indexing_master.py | 31 +--------- .../package/scripts/params/params_linux.py | 5 ++ metron-platform/metron-solr/README.md | 33 +++++++++- .../src/main/scripts/create_collection.sh | 21 +++++-- .../src/main/scripts/delete_collection.sh | 18 ++++-- 6 files changed, 120 insertions(+), 51 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/f241f87f/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py index eeb2127..69e980b 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py @@ -25,6 +25,7 @@ from resource_management.core.exceptions import Fail from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute, File from resource_management.libraries.functions import format as ambari_format +from resource_management.libraries.functions.format import format import metron_service import metron_security @@ -85,14 +86,13 @@ class IndexingCommands: :return: Dict where key is the name of a collection and the value is a path to file containing the schema definition. """ - from params import params - return { - "bro": params.bro_schema_path, - "yaf": params.yaf_schema_path, - "snort": params.snort_schema_path, - "error": params.error_schema_path, - "metaalert": params.meta_schema_path - } + return [ + "bro", + "yaf", + "snort", + "error", + "metaalert" + ] def is_configured(self): return self.__configured @@ -200,6 +200,53 @@ class IndexingCommands: user=self.__params.metron_user, err_msg=err_msg.format(template_name)) + def solr_schema_install(self, env): + from params import params + env.set_params(params) + Logger.info("Installing Solr schemas") + if self.__params.security_enabled: + metron_security.kinit(self.__params.kinit_path_local, + self.__params.solr_keytab_path, + self.__params.solr_principal_name, + self.__params.solr_user) + + commands = IndexingCommands(params) + for collection_name in commands.get_solr_schemas(): + + # install the schema + cmd = format(( + "export ZOOKEEPER={solr_zookeeper_url};" + "export SECURITY_ENABLED={security_enabled};" + )) + cmd += "{0}/bin/create_collection.sh {1};" + + Execute( + cmd.format(params.metron_home, collection_name), + user=self.__params.solr_user) + + def solr_schema_delete(self, env): + from params import params + env.set_params(params) + Logger.info("Deleting Solr schemas") + if self.__params.security_enabled: + metron_security.kinit(self.__params.kinit_path_local, + self.__params.solr_keytab_path, + self.__params.solr_principal_name, + self.__params.solr_user) + + commands = IndexingCommands(params) + for collection_name in commands.get_solr_schemas(): + # delete the schema + cmd = format(( + "export ZOOKEEPER={solr_zookeeper_url};" + "export SECURITY_ENABLED={security_enabled};" + )) + cmd += "{0}/bin/delete_collection.sh {1};" + + Execute( + cmd.format(params.metron_home, collection_name), + user=self.__params.solr_user) + def start_batch_indexing_topology(self, env): Logger.info('Starting ' + self.__batch_indexing_topology) http://git-wip-us.apache.org/repos/asf/metron/blob/f241f87f/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py index b5c4bb9..1629465 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py @@ -97,11 +97,10 @@ class Indexing(Script): self.configure(env) commands = IndexingCommands(params) if params.ra_indexing_writer == 'Solr': - Logger.info("Loading Solr schemas") # Install Solr schemas try: if not commands.is_solr_schema_installed(): - self.solr_schema_install(env) + commands.solr_schema_install(env) commands.set_solr_schema_installed() except Exception as e: @@ -170,34 +169,6 @@ class Indexing(Script): cmd.format(params.es_http_url, template_name), logoutput=True) - def solr_schema_install(self, env): - from params import params - env.set_params(params) - Logger.info("Installing Solr schemas") - - commands = IndexingCommands(params) - for collection_name, config_path in commands.get_solr_schemas().iteritems(): - - # install the schema - - cmd = "{0}/bin/solr create -c {1} -d {2}" - Execute( - cmd.format(params.solr_home, collection_name, config_path), - logoutput=True, user="solr") - - def solr_schema_delete(self, env): - from params import params - env.set_params(params) - Logger.info("Deleting Solr schemas") - - commands = IndexingCommands(params) - for collection_name, config_path in commands.get_solr_schemas().iteritems(): - # delete the schema - cmd = "{0}/bin/solr delete -c {1}" - Execute( - cmd.format(params.solr_home, collection_name), - logoutput=True, user="solr") - @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) def kibana_dashboard_install(self, env): from params import params http://git-wip-us.apache.org/repos/asf/metron/blob/f241f87f/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index 6f4760b..667a926 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -128,6 +128,9 @@ if has_zk_host: solr_version = '6.6.2' solr_home = '/var/solr/solr-' + solr_version solr_zookeeper_url = format(format(config['configurations']['metron-env']['solr_zookeeper_url'])) +solr_user = config['configurations']['solr-config-env']['solr_config_user'] +solr_principal_name = config['configurations']['solr-config-env']['solr_principal_name'] +solr_keytab_path = config['configurations']['solr-config-env']['solr_keytab_path'] # Storm storm_rest_addr = status_params.storm_rest_addr @@ -256,6 +259,8 @@ if security_enabled: nimbus_seeds = config['configurations']['storm-site']['nimbus.seeds'] + solr_principal_name = solr_principal_name.replace('_HOST', hostname_lowercase) + # Management UI metron_rest_host = default("/clusterHostInfo/metron_rest_hosts", [hostname])[0] http://git-wip-us.apache.org/repos/asf/metron/blob/f241f87f/metron-platform/metron-solr/README.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/README.md b/metron-platform/metron-solr/README.md index 0d525ec..159779c 100644 --- a/metron-platform/metron-solr/README.md +++ b/metron-platform/metron-solr/README.md @@ -20,7 +20,10 @@ limitations under the License. ## Table of Contents * [Introduction](#introduction) +* [Configuration](#configuration) * [Installing](#installing) +* [Schemas](#schemas) +* [Collections](#collections) ## Introduction @@ -110,4 +113,32 @@ A PointType field should be defined as: <dynamicField name="*_point" type="pdouble" indexed="true" stored="false" docValues="false"/> <fieldType name="point" class="solr.PointType" subFieldSuffix="_point"/> ``` -If any copy fields are defined, stored and docValues should be set to false. \ No newline at end of file +If any copy fields are defined, stored and docValues should be set to false. + +## Collections + +Convenience scripts are provided with Metron to create and delete collections. Ambari uses these scripts to automatically create collections. To use them outside of Ambari, a few environment variables must be set first: +``` +# Path to the zookeeper node used by Solr +export ZOOKEEPER=node1:2181/solr +# Set to true if Kerberos is enabled +export SECURITY_ENABLED=true +``` +The scripts can then be called directly with the collection name as the first argument . For example, to create the bro collection: +``` +$METRON_HOME/bin/create_collection.sh bro +``` +To delete the bro collection: +``` +$METRON_HOME/bin/delete_collection.sh bro +``` +The `create_collection.sh` script depends on schemas installed in `$METRON_HOME/config/schema`. There are several schemas that come with Metron: + +* bro +* snort +* yaf +* metaalert +* error + +Additional schemas should be installed in that location if using the `create_collection.sh` script. Any collection can be deleted with the `delete_collection.sh` script. +These scripts use the [Solr Collection API](http://lucene.apache.org/solr/guide/6_6/collections-api.html). \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/f241f87f/metron-platform/metron-solr/src/main/scripts/create_collection.sh ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/scripts/create_collection.sh b/metron-platform/metron-solr/src/main/scripts/create_collection.sh index 37fbccb..7693646 100755 --- a/metron-platform/metron-solr/src/main/scripts/create_collection.sh +++ b/metron-platform/metron-solr/src/main/scripts/create_collection.sh @@ -18,10 +18,19 @@ # METRON_VERSION=${project.version} METRON_HOME=/usr/metron/$METRON_VERSION -SOLR_VERSION=${global_solr_version} -SOLR_USER=solr -SOLR_SERVICE=$SOLR_USER -SOLR_VAR_DIR="/var/$SOLR_SERVICE" +ZOOKEEPER=${ZOOKEEPER:-localhost:2181} +ZOOKEEPER_HOME=${ZOOKEEPER_HOME:-/usr/hdp/current/zookeeper-client} +SECURITY_ENABLED=${SECURITY_ENABLED:-false} +NEGOTIATE='' +if [ ${SECURITY_ENABLED,,} == 'true' ]; then + NEGOTIATE=' --negotiate -u : ' +fi -cd $SOLR_VAR_DIR/solr-${SOLR_VERSION} -su $SOLR_USER -c "bin/solr create -c $1 -d $METRON_HOME/config/schema/$1/" +# Get the first Solr node from the list of live nodes in Zookeeper +SOLR_NODE=`$ZOOKEEPER_HOME/bin/zkCli.sh -server $ZOOKEEPER ls /live_nodes | tail -n 1 | sed 's/\[\([^,]*\).*\]/\1/' | sed 's/_solr//'` + +# Upload the collection config set +zip -rj - $METRON_HOME/config/schema/$1 | curl -X POST $NEGOTIATE --header "Content-Type:text/xml" --data-binary @- "http://$SOLR_NODE/solr/admin/configs?action=UPLOAD&name=$1" + +# Create the collection +curl -X GET $NEGOTIATE "http://$SOLR_NODE/solr/admin/collections?action=CREATE&name=$1&numShards=1" http://git-wip-us.apache.org/repos/asf/metron/blob/f241f87f/metron-platform/metron-solr/src/main/scripts/delete_collection.sh ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/scripts/delete_collection.sh b/metron-platform/metron-solr/src/main/scripts/delete_collection.sh index 68f0c20..c8b45e7 100755 --- a/metron-platform/metron-solr/src/main/scripts/delete_collection.sh +++ b/metron-platform/metron-solr/src/main/scripts/delete_collection.sh @@ -18,10 +18,16 @@ # METRON_VERSION=${project.version} METRON_HOME=/usr/metron/$METRON_VERSION -SOLR_VERSION=${global_solr_version} -SOLR_USER=solr -SOLR_SERVICE=$SOLR_USER -SOLR_VAR_DIR="/var/$SOLR_SERVICE" +ZOOKEEPER=${ZOOKEEPER:-localhost:2181} +ZOOKEEPER_HOME=${ZOOKEEPER_HOME:-/usr/hdp/current/zookeeper-client} +SECURITY_ENABLED=${SECURITY_ENABLED:-false} +NEGOTIATE='' +if [ ${SECURITY_ENABLED,,} == 'true' ]; then + NEGOTIATE=' --negotiate -u : ' +fi -cd $SOLR_VAR_DIR/solr-${SOLR_VERSION} -su $SOLR_USER -c "bin/solr delete -c $1" +# Get the first Solr node from the list of live nodes in Zookeeper +SOLR_NODE=`$ZOOKEEPER_HOME/bin/zkCli.sh -server $ZOOKEEPER ls /live_nodes | tail -n 1 | sed 's/\[\([^,]*\).*\]/\1/' | sed 's/_solr//'` + +# Delete the collection +curl -X GET $NEGOTIATE "http://$SOLR_NODE/solr/admin/collections?action=DELETE&name=$1"