Repository: metron Updated Branches: refs/heads/master 6f2679919 -> 0c6aad8c7
METRON-1410 Some more upgrade fallout... Can't restart Metron Indexing. (ottobackwards) closes apache/metron#901 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0c6aad8c Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0c6aad8c Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0c6aad8c Branch: refs/heads/master Commit: 0c6aad8c7c7affd6c898e1dd7663833a71ba63de Parents: 6f26799 Author: ottobackwards <ottobackwa...@gmail.com> Authored: Sat Jan 27 17:22:46 2018 -0500 Committer: otto <o...@apache.org> Committed: Sat Jan 27 17:22:46 2018 -0500 ---------------------------------------------------------------------- .../package/scripts/enrichment_commands.py | 36 +++++--------------- .../package/scripts/indexing_commands.py | 18 ++-------- .../CURRENT/package/scripts/metron_service.py | 26 ++++++++++++++ .../package/scripts/profiler_commands.py | 26 ++++---------- 4 files changed, 45 insertions(+), 61 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/0c6aad8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py index 90a690e..f9ec547 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py @@ -17,14 +17,15 @@ limitations under the License. import os import time -from datetime import datetime + from resource_management.core.exceptions import Fail from resource_management.core.logger import Logger -from resource_management.core.resources.system import Execute, File +from resource_management.core.resources.system import Execute import metron_service from metron_security import kinit + # Wrap major operations and functionality in this class class EnrichmentCommands: __params = None @@ -184,31 +185,12 @@ class EnrichmentCommands: def create_hbase_tables(self): Logger.info("Creating HBase Tables") - if self.__params.security_enabled: - kinit(self.__params.kinit_path_local, - self.__params.hbase_keytab_path, - self.__params.hbase_principal_name, - execute_user=self.__params.hbase_user) - - cmd = "echo \"create '{0}','{1}'\" | hbase shell -n" - add_enrichment_cmd = cmd.format(self.__params.enrichment_hbase_table, self.__params.enrichment_hbase_cf) - Execute(add_enrichment_cmd, - tries=3, - try_sleep=5, - logoutput=False, - path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', - user=self.__params.hbase_user - ) - - add_threatintel_cmd = cmd.format(self.__params.threatintel_hbase_table, self.__params.threatintel_hbase_cf) - Execute(add_threatintel_cmd, - tries=3, - try_sleep=5, - logoutput=False, - path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', - user=self.__params.hbase_user - ) - + metron_service.create_hbase_table(self.__params, + self.__params.enrichment_hbase_table, + self.__params.enrichment_hbase_cf) + metron_service.create_hbase_table(self.__params, + self.__params.threatintel_hbase_table, + self.__params.threatintel_hbase_cf) Logger.info("Done creating HBase Tables") self.set_hbase_configured() http://git-wip-us.apache.org/repos/asf/metron/blob/0c6aad8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py index 33f45d4..c057b72 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py @@ -113,21 +113,9 @@ class IndexingCommands: def create_hbase_tables(self): Logger.info("Creating HBase Tables for indexing") - if self.__params.security_enabled: - metron_security.kinit(self.__params.kinit_path_local, - self.__params.hbase_keytab_path, - self.__params.hbase_principal_name, - execute_user=self.__params.hbase_user) - cmd = "echo \"create '{0}','{1}'\" | hbase shell -n" - add_update_cmd = cmd.format(self.__params.update_hbase_table, self.__params.update_hbase_cf) - Execute(add_update_cmd, - tries=3, - try_sleep=5, - logoutput=False, - path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', - user=self.__params.hbase_user - ) - + metron_service.create_hbase_table(self.__params, + self.__params.update_hbase_table, + self.__params.update_hbase_cf) Logger.info("Done creating HBase Tables for indexing") self.set_hbase_configured() http://git-wip-us.apache.org/repos/asf/metron/blob/0c6aad8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py index d8b8b10..330d3c0 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py @@ -282,6 +282,32 @@ def check_kafka_topics(params, topics): err_msg = "Missing Kafka topic; topic={0}".format(topic) execute(cmd, user=params.kafka_user, err_msg=err_msg) + +def create_hbase_table(params, table, cf): + """ + Creates an HBase table, if the table does not currently exist + :param params: + :param table: The name of the HBase table. + :param cf: The column family + :param user: The user to execute the command as + """ + if params.security_enabled: + kinit(params.kinit_path_local, + params.hbase_keytab_path, + params.hbase_principal_name, + execute_user=params.hbase_user) + cmd = """if [[ $(echo \"exists '{0}'\" | hbase shell | grep 'not exist') ]]; \ + then echo \"create '{0}','{1}'\" | hbase shell -n; fi""" + add_update_cmd = cmd.format(table, cf) + Execute(add_update_cmd, + tries=3, + try_sleep=5, + logoutput=False, + path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', + user=params.hbase_user + ) + + def check_hbase_table(params, table): """ Validates that an HBase table exists. An exception is raised if the table http://git-wip-us.apache.org/repos/asf/metron/blob/0c6aad8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py index 41cab06..b026a30 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py @@ -18,13 +18,12 @@ limitations under the License. import os import time -from datetime import datetime from resource_management.core.exceptions import Fail from resource_management.core.logger import Logger -from resource_management.core.resources.system import Execute, File +from resource_management.core.resources.system import Execute -import metron_service import metron_security +import metron_service # Wrap major operations and functionality in this class @@ -79,22 +78,11 @@ class ProfilerCommands: metron_service.set_configured(self.__params.metron_user, self.__params.profiler_hbase_acl_configured_flag_file, "Setting HBase ACL configured to True for profiler") def create_hbase_tables(self): - Logger.info("Creating HBase table '{0}' for profiler".format(self.__params.profiler_hbase_table)) - if self.__params.security_enabled: - metron_security.kinit(self.__params.kinit_path_local, - self.__params.hbase_keytab_path, - self.__params.hbase_principal_name, - execute_user=self.__params.hbase_user) - cmd = "echo \"create '{0}','{1}'\" | hbase shell -n" - add_table_cmd = cmd.format(self.__params.profiler_hbase_table, self.__params.profiler_hbase_cf) - Execute(add_table_cmd, - tries=3, - try_sleep=5, - logoutput=False, - path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', - user=self.__params.hbase_user - ) - + Logger.info("Creating HBase table '{0}' for profiler".format( + self.__params.profiler_hbase_table)) + metron_service.create_hbase_table(self.__params, + self.__params.profiler_hbase_table, + self.__params.profiler_hbase_cf) self.set_hbase_configured() Logger.info("Done creating HBase Tables for profiler")
