This is an automated email from the ASF dual-hosted git repository. vjasani pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ambari-infra.git
The following commit(s) were added to refs/heads/master by this push: new 26f3717 AMBARI-26021: Upgrade Ambari Infra to python3 (#53) 26f3717 is described below commit 26f3717959956dbdd4d7a53ebc7d6a01d98dd5b8 Author: jialiang <2510095...@qq.com> AuthorDate: Thu Apr 25 07:00:11 2024 +0800 AMBARI-26021: Upgrade Ambari Infra to python3 (#53) --- .../src/main/python/migrationConfigGenerator.py | 70 ++--- .../src/main/python/migrationHelper.py | 332 ++++++++++----------- .../src/main/python/solrDataManager.py | 126 ++++---- 3 files changed, 264 insertions(+), 264 deletions(-) diff --git a/ambari-infra-solr-client/src/main/python/migrationConfigGenerator.py b/ambari-infra-solr-client/src/main/python/migrationConfigGenerator.py index 125b59d..8166e3f 100755 --- a/ambari-infra-solr-client/src/main/python/migrationConfigGenerator.py +++ b/ambari-infra-solr-client/src/main/python/migrationConfigGenerator.py @@ -24,12 +24,12 @@ import signal import sys import time import traceback -import urllib2, ssl +import urllib.request, urllib.error, urllib.parse, ssl import logging import json import base64 import optparse -import ConfigParser +import configparser from subprocess import Popen, PIPE from random import randrange @@ -71,7 +71,7 @@ def api_accessor(host, username, password, protocol, port): if request_body: logger.debug('Request body: {0}'.format(request_body)) admin_auth = base64.encodestring('%s:%s' % (username, password)).replace('\n', '') - request = urllib2.Request(url) + request = urllib.request.Request(url) request.add_header('Authorization', 'Basic %s' % admin_auth) request.add_header('X-Requested-By', 'ambari') request.add_data(request_body) @@ -81,9 +81,9 @@ def api_accessor(host, username, password, protocol, port): ctx = ssl.create_default_context() ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE - response = urllib2.urlopen(request, context=ctx) + response = urllib.request.urlopen(request, context=ctx) else: - response = urllib2.urlopen(request) + response = urllib.request.urlopen(request) response_body = response.read() except Exception as exc: raise Exception('Problem with accessing api. Reason: {0}'.format(exc)) @@ -121,7 +121,7 @@ def retry(func, *args, **kwargs): def get_shard_numbers_per_collections(state_json_data): collection_shard_map={} - for key,val in state_json_data.iteritems(): + for key,val in state_json_data.items(): if 'shards' in val: shard_count=len(val['shards']) collection_shard_map[key]=shard_count @@ -129,7 +129,7 @@ def get_shard_numbers_per_collections(state_json_data): def get_max_shards_for_collections(state_json_data): collection_max_shard_map={} - for key,val in state_json_data.iteritems(): + for key,val in state_json_data.items(): if 'maxShardsPerNode' in val: collection_max_shard_map[key]=val['maxShardsPerNode'] return collection_max_shard_map @@ -261,9 +261,9 @@ def get_installed_components(blueprint): def generate_ambari_solr_migration_ini_file(options, accessor, protocol): - print "Start generating config file: {0} ...".format(options.ini_file) + print("Start generating config file: {0} ...".format(options.ini_file)) - config = ConfigParser.RawConfigParser() + config = configparser.RawConfigParser() config.add_section('ambari_server') config.set('ambari_server', 'host', options.host) @@ -273,34 +273,34 @@ def generate_ambari_solr_migration_ini_file(options, accessor, protocol): config.set('ambari_server', 'username', options.username) config.set('ambari_server', 'password', options.password) - print "Get Ambari cluster details ..." + print("Get Ambari cluster details ...") blueprint = get_json(accessor, CLUSTERS_URL.format(options.cluster) + BLUEPRINT_CONFIG_URL) installed_components = get_installed_components(blueprint) - print "Set JAVA_HOME: {0}".format(options.java_home) + print("Set JAVA_HOME: {0}".format(options.java_home)) host = socket.getfqdn() cluster_config = get_cluster_configs(blueprint) solr_hosts = get_solr_hosts(options, accessor) if solr_hosts and host not in solr_hosts: - print "{0}WARNING{1}: Host '{2}' is not found in Infra Solr hosts ({3}). Migration commands won't work from here."\ - .format(colors.WARNING, colors.ENDC, host, ','.join(solr_hosts)) + print("{0}WARNING{1}: Host '{2}' is not found in Infra Solr hosts ({3}). Migration commands won't work from here."\ + .format(colors.WARNING, colors.ENDC, host, ','.join(solr_hosts))) zookeeper_hosts = get_zookeeper_server_hosts(options, accessor) security_enabled = is_security_enabled(cluster_config) zk_connect_string = get_zookeeper_connection_string(cluster_config, zookeeper_hosts) if zk_connect_string: - print "Service detected: " + colors.OKGREEN + "ZOOKEEPER" + colors.ENDC - print "Zookeeper connection string: {0}".format(str(zk_connect_string)) + print("Service detected: " + colors.OKGREEN + "ZOOKEEPER" + colors.ENDC) + print("Zookeeper connection string: {0}".format(str(zk_connect_string))) solr_protocol = get_solr_protocol(cluster_config) solr_urls = get_solr_urls(cluster_config, solr_hosts, solr_protocol) if solr_urls: - print "Service detected: " + colors.OKGREEN + "AMBARI_INFRA_SOLR" + colors.ENDC + print("Service detected: " + colors.OKGREEN + "AMBARI_INFRA_SOLR" + colors.ENDC) solr_znode = get_solr_znode(cluster_config) if solr_znode: - print "Infra Solr znode: {0}".format(solr_znode) + print("Infra Solr znode: {0}".format(solr_znode)) infra_solr_env_props = get_config_props(cluster_config, 'infra-solr-env') infra_solr_user = infra_solr_env_props['infra_solr_user'] if 'infra_solr_user' in infra_solr_env_props else 'infra-solr' @@ -340,7 +340,7 @@ def generate_ambari_solr_migration_ini_file(options, accessor, protocol): default_zk_quorum = "{zookeeper_quorum}" external_zk_connection_string = infra_solr_env_props['infra_solr_zookeeper_quorum'] if 'infra_solr_zookeeper_quorum' in infra_solr_env_props else default_zk_quorum if default_zk_quorum != external_zk_connection_string: - print "Found external zk connection string: {0}".format(external_zk_connection_string) + print("Found external zk connection string: {0}".format(external_zk_connection_string)) config.set('infra_solr', 'external_zk_connect_string', external_zk_connection_string) config.set('infra_solr', 'zk_principal_user', zk_principal_user) @@ -350,7 +350,7 @@ def generate_ambari_solr_migration_ini_file(options, accessor, protocol): config.add_section('ranger_collection') if "RANGER_ADMIN" in installed_components and not options.skip_ranger: - print "Service detected: " + colors.OKGREEN + "RANGER" + colors.ENDC + print("Service detected: " + colors.OKGREEN + "RANGER" + colors.ENDC) ranger_env_props = get_config_props(cluster_config, 'ranger-env') if "is_solrCloud_enabled" in ranger_env_props and ranger_env_props['is_solrCloud_enabled'] == 'true': if "is_external_solrCloud_enabled" in ranger_env_props and ranger_env_props['is_external_solrCloud_enabled'] == 'true' and not options.force_ranger: @@ -367,7 +367,7 @@ def generate_ambari_solr_migration_ini_file(options, accessor, protocol): config.set('ranger_collection', 'ranger_collection_max_shards_per_node', max_shards_map[ranger_collection_name]) config.set('ranger_collection', 'backup_ranger_config_set_name', 'old_ranger_audits') config.set('ranger_collection', 'backup_ranger_collection_name', 'old_ranger_audits') - print 'Ranger Solr collection: ' + ranger_collection_name + print('Ranger Solr collection: ' + ranger_collection_name) ranger_backup_path = None if options.backup_base_path: ranger_backup_path = os.path.join(options.backup_base_path, "ranger") @@ -375,7 +375,7 @@ def generate_ambari_solr_migration_ini_file(options, accessor, protocol): ranger_backup_path = options.backup_ranger_base_path if ranger_backup_path is not None: config.set('ranger_collection', 'backup_path', ranger_backup_path) - print 'Ranger backup path: ' + ranger_backup_path + print('Ranger backup path: ' + ranger_backup_path) if options.ranger_hdfs_base_path: config.set('ranger_collection', 'hdfs_base_path', options.ranger_hdfs_base_path) elif options.hdfs_base_path: @@ -387,7 +387,7 @@ def generate_ambari_solr_migration_ini_file(options, accessor, protocol): config.add_section('atlas_collections') if "ATLAS_SERVER" in installed_components and not options.skip_atlas: - print "Service detected: " + colors.OKGREEN + "ATLAS" + colors.ENDC + print("Service detected: " + colors.OKGREEN + "ATLAS" + colors.ENDC) config.set('atlas_collections', 'enabled', 'true') config.set('atlas_collections', 'config_set', 'atlas_configs') config.set('atlas_collections', 'fulltext_index_name', 'fulltext_index') @@ -408,7 +408,7 @@ def generate_ambari_solr_migration_ini_file(options, accessor, protocol): config.set('atlas_collections', 'vertex_index_shards', coll_shard_map['vertex_index']) if 'vertex_index' in max_shards_map: config.set('atlas_collections', 'vertex_index_max_shards_per_node', max_shards_map['vertex_index']) - print 'Atlas Solr collections: fulltext_index, edge_index, vertex_index' + print('Atlas Solr collections: fulltext_index, edge_index, vertex_index') atlas_backup_path = None if options.backup_base_path: atlas_backup_path = os.path.join(options.backup_base_path, "atlas") @@ -416,7 +416,7 @@ def generate_ambari_solr_migration_ini_file(options, accessor, protocol): atlas_backup_path = options.backup_atlas_base_path if atlas_backup_path is not None: config.set('atlas_collections', 'backup_path', atlas_backup_path) - print 'Atlas backup path: ' + atlas_backup_path + print('Atlas backup path: ' + atlas_backup_path) if options.atlas_hdfs_base_path: config.set('atlas_collections', 'hdfs_base_path', options.atlas_hdfs_base_path) elif options.hdfs_base_path: @@ -426,7 +426,7 @@ def generate_ambari_solr_migration_ini_file(options, accessor, protocol): config.add_section('logsearch_collections') if "LOGSEARCH_SERVER" in installed_components: - print "Service detected: " + colors.OKGREEN + "LOGSEARCH" + colors.ENDC + print("Service detected: " + colors.OKGREEN + "LOGSEARCH" + colors.ENDC) logsearch_props = get_config_props(cluster_config, 'logsearch-properties') @@ -437,19 +437,19 @@ def generate_ambari_solr_migration_ini_file(options, accessor, protocol): config.set('logsearch_collections', 'hadoop_logs_collection_name', logsearch_hadoop_logs_coll_name) config.set('logsearch_collections', 'audit_logs_collection_name', logsearch_audit_logs_coll_name) config.set('logsearch_collections', 'history_collection_name', 'history') - print 'Log Search Solr collections: {0}, {1}, history'.format(logsearch_hadoop_logs_coll_name, logsearch_audit_logs_coll_name) + print('Log Search Solr collections: {0}, {1}, history'.format(logsearch_hadoop_logs_coll_name, logsearch_audit_logs_coll_name)) else: config.set('logsearch_collections', 'enabled', 'false') if security_enabled == 'true': - print "Kerberos: enabled" + print("Kerberos: enabled") else: - print "Kerberos: disabled" + print("Kerberos: disabled") with open(options.ini_file, 'w') as f: config.write(f) - print "Config file generation has finished " + colors.OKGREEN + "successfully" + colors.ENDC + print("Config file generation has finished " + colors.OKGREEN + "successfully" + colors.ENDC) def validate_inputs(options): errors=[] @@ -500,10 +500,10 @@ if __name__=="__main__": errors = validate_inputs(options) if errors: - print 'Errors' + print('Errors') for error in errors: - print '- {0}'.format(error) - print '' + print('- {0}'.format(error)) + print('') parser.print_help() else: protocol = 'https' if options.ssl else 'http' @@ -511,8 +511,8 @@ if __name__=="__main__": try: generate_ambari_solr_migration_ini_file(options, accessor, protocol) except Exception as exc: - print traceback.format_exc() - print 'Config file generation ' + colors.FAIL + 'failed' + colors.ENDC + print(traceback.format_exc()) + print('Config file generation ' + colors.FAIL + 'failed' + colors.ENDC) except KeyboardInterrupt: - print + print() sys.exit(128 + signal.SIGINT) \ No newline at end of file diff --git a/ambari-infra-solr-client/src/main/python/migrationHelper.py b/ambari-infra-solr-client/src/main/python/migrationHelper.py index a257b34..62d186e 100755 --- a/ambari-infra-solr-client/src/main/python/migrationHelper.py +++ b/ambari-infra-solr-client/src/main/python/migrationHelper.py @@ -23,14 +23,14 @@ import glob import logging import os import sys -import urllib2, ssl +import urllib.request, urllib.error, urllib.parse, ssl import json import base64 import optparse import socket import time import traceback -import ConfigParser +import configparser import solrDataManager as solr_data_manager from datetime import datetime, timedelta @@ -98,7 +98,7 @@ def api_accessor(host, username, password, protocol, port): if request_body: logger.debug('Request body: {0}'.format(request_body)) admin_auth = base64.encodestring('%s:%s' % (username, password)).replace('\n', '') - request = urllib2.Request(url) + request = urllib.request.Request(url) request.add_header('Authorization', 'Basic %s' % admin_auth) request.add_header('X-Requested-By', 'ambari') request.add_data(request_body) @@ -108,9 +108,9 @@ def api_accessor(host, username, password, protocol, port): ctx = ssl.create_default_context() ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE - response = urllib2.urlopen(request, context=ctx) + response = urllib.request.urlopen(request, context=ctx) else: - response = urllib2.urlopen(request) + response = urllib.request.urlopen(request) response_body = response.read() except Exception as exc: raise Exception('Problem with accessing api. Reason: {0}'.format(exc)) @@ -135,7 +135,7 @@ def retry(func, *args, **kwargs): logger.error("Error occurred during {0} operation: {1}".format(context, str(traceback.format_exc()))) logger.info("\n{0}: waiting for {1} seconds before retyring again (retry count: {2})".format(context, delay, r+1)) time.sleep(delay) - print '{0} operation {1}FAILED{2}'.format(context, colors.FAIL, colors.ENDC) + print('{0} operation {1}FAILED{2}'.format(context, colors.FAIL, colors.ENDC)) sys.exit(1) def get_keytab_and_principal(config): @@ -223,7 +223,7 @@ def get_random_solr_url(solr_urls, options = None): def format_json(dictionary, tab_level=0): output = '' tab = ' ' * 2 * tab_level - for key, value in dictionary.iteritems(): + for key, value in dictionary.items(): output += ',\n{0}"{1}": '.format(tab, key) if isinstance(value, dict): output += '{\n' + format_json(value, tab_level + 1) + tab + '}' @@ -371,7 +371,7 @@ def fill_parameters(options, config, collection, index_location, hdfs_path=None, params['solr_index_version'] = options.index_version if options.force: params['solr_index_upgrade_force'] = options.force - if options.async: + if options.async_op: params['solr_request_async'] = options.request_async if options.request_tries: params['solr_request_tries'] = options.request_tries @@ -398,12 +398,12 @@ def fill_parameters(options, config, collection, index_location, hdfs_path=None, def validte_common_options(options, parser, config): if not options.index_location: parser.print_help() - print 'index-location option is required' + print('index-location option is required') sys.exit(1) if not options.collection: parser.print_help() - print 'collection option is required' + print('collection option is required') sys.exit(1) def get_service_components(options, accessor, cluster, service, component): @@ -484,16 +484,16 @@ def upgrade_solr_instances(options, accessor, parser, config): request_id = get_request_id(response) sys.stdout.write(colors.OKGREEN + 'DONE\n' + colors.ENDC) sys.stdout.flush() - print 'Upgrade command request id: {0}'.format(request_id) - if options.async: - print "Upgrade request sent to Ambari server. Check Ambari UI about the results." + print('Upgrade command request id: {0}'.format(request_id)) + if options.async_op: + print("Upgrade request sent to Ambari server. Check Ambari UI about the results.") sys.exit(0) else: sys.stdout.write("Start monitoring Ambari request with id {0} ...".format(request_id)) sys.stdout.flush() cluster = config.get('ambari_server', 'cluster') monitor_request(options, accessor, cluster, request_id, context) - print "{0}... {1} DONE{2}".format(context, colors.OKGREEN, colors.ENDC) + print("{0}... {1} DONE{2}".format(context, colors.OKGREEN, colors.ENDC)) def upgrade_solr_clients(options, accessor, parser, config): """ @@ -510,7 +510,7 @@ def upgrade_solr_clients(options, accessor, parser, config): solr_client_hosts.remove(host) if not len(solr_client_hosts): - print 'The Solr Clients upgrade request has been aborted because no other host can be upgraded.' + print('The Solr Clients upgrade request has been aborted because no other host can be upgraded.') sys.exit(0) context = "Upgrade Solr Clients" @@ -522,16 +522,16 @@ def upgrade_solr_clients(options, accessor, parser, config): request_id = get_request_id(response) sys.stdout.write(colors.OKGREEN + 'DONE\n' + colors.ENDC) sys.stdout.flush() - print 'Upgrade command request id: {0}'.format(request_id) - if options.async: - print "Upgrade request sent to Ambari server. Check Ambari UI about the results." + print('Upgrade command request id: {0}'.format(request_id)) + if options.async_op: + print("Upgrade request sent to Ambari server. Check Ambari UI about the results.") sys.exit(0) else: sys.stdout.write("Start monitoring Ambari request with id {0} ...".format(request_id)) sys.stdout.flush() cluster = config.get('ambari_server', 'cluster') monitor_request(options, accessor, cluster, request_id, context) - print "{0}... {1}DONE{2}".format(context, colors.OKGREEN, colors.ENDC) + print("{0}... {1}DONE{2}".format(context, colors.OKGREEN, colors.ENDC)) def upgrade_logfeeders(options, accessor, parser, config): """ @@ -549,16 +549,16 @@ def upgrade_logfeeders(options, accessor, parser, config): request_id = get_request_id(response) sys.stdout.write(colors.OKGREEN + 'DONE\n' + colors.ENDC) sys.stdout.flush() - print 'Upgrade command request id: {0}'.format(request_id) - if options.async: - print "Upgrade request sent to Ambari server. Check Ambari UI about the results." + print('Upgrade command request id: {0}'.format(request_id)) + if options.async_op: + print("Upgrade request sent to Ambari server. Check Ambari UI about the results.") sys.exit(0) else: sys.stdout.write("Start monitoring Ambari request with id {0} ...".format(request_id)) sys.stdout.flush() cluster = config.get('ambari_server', 'cluster') monitor_request(options, accessor, cluster, request_id, context) - print "{0}... {1} DONE{2}".format(context, colors.OKGREEN, colors.ENDC) + print("{0}... {1} DONE{2}".format(context, colors.OKGREEN, colors.ENDC)) def upgrade_logsearch_portal(options, accessor, parser, config): """ @@ -576,16 +576,16 @@ def upgrade_logsearch_portal(options, accessor, parser, config): request_id = get_request_id(response) sys.stdout.write(colors.OKGREEN + 'DONE\n' + colors.ENDC) sys.stdout.flush() - print 'Upgrade command request id: {0}'.format(request_id) - if options.async: - print "Upgrade request sent to Ambari server. Check Ambari UI about the results." + print('Upgrade command request id: {0}'.format(request_id)) + if options.async_op: + print("Upgrade request sent to Ambari server. Check Ambari UI about the results.") sys.exit(0) else: sys.stdout.write("Start monitoring Ambari request with id {0} ...".format(request_id)) sys.stdout.flush() cluster = config.get('ambari_server', 'cluster') monitor_request(options, accessor, cluster, request_id, context) - print "{0}... {1} DONE{2}".format(context, colors.OKGREEN, colors.ENDC) + print("{0}... {1} DONE{2}".format(context, colors.OKGREEN, colors.ENDC)) def service_components_command(options, accessor, parser, config, service, component, command, command_str): """ @@ -603,16 +603,16 @@ def service_components_command(options, accessor, parser, config, service, compo request_id = get_request_id(response) sys.stdout.write(colors.OKGREEN + 'DONE\n' + colors.ENDC) sys.stdout.flush() - print '{0} command request id: {1}'.format(command_str, request_id) - if options.async: - print "{0} request sent to Ambari server. Check Ambari UI about the results.".format(command_str) + print('{0} command request id: {1}'.format(command_str, request_id)) + if options.async_op: + print("{0} request sent to Ambari server. Check Ambari UI about the results.".format(command_str)) sys.exit(0) else: sys.stdout.write("Start monitoring Ambari request with id {0} ...".format(request_id)) sys.stdout.flush() cluster = config.get('ambari_server', 'cluster') monitor_request(options, accessor, cluster, request_id, context) - print "{0}... {1} DONE{2}".format(context, colors.OKGREEN, colors.ENDC) + print("{0}... {1} DONE{2}".format(context, colors.OKGREEN, colors.ENDC)) def monitor_request(options, accessor, cluster, request_id, context): while True: @@ -622,10 +622,10 @@ def monitor_request(options, accessor, cluster, request_id, context): logger.debug("\nMonitoring '{0}' request (id: '{1}') status is {2}".format(context, request_id, request_status)) if request_status in ['FAILED', 'TIMEDOUT', 'ABORTED', 'COMPLETED', 'SKIPPED_FAILED']: if request_status == 'COMPLETED': - print "\nRequest (id: {0}) {1}COMPLETED{2}".format(request_id, colors.OKGREEN, colors.ENDC) + print("\nRequest (id: {0}) {1}COMPLETED{2}".format(request_id, colors.OKGREEN, colors.ENDC)) time.sleep(4) else: - print "\nRequest (id: {0}) {1}FAILED{2} (checkout Ambari UI about the failed tasks)\n".format(request_id, colors.FAIL, colors.ENDC) + print("\nRequest (id: {0}) {1}FAILED{2} (checkout Ambari UI about the failed tasks)\n".format(request_id, colors.FAIL, colors.ENDC)) sys.exit(1) break else: @@ -635,7 +635,7 @@ def monitor_request(options, accessor, cluster, request_id, context): logger.debug("Sleep 5 seconds ...") time.sleep(5) else: - print "'Requests' or 'request_status' cannot be found in JSON response: {0}".format(request_response) + print("'Requests' or 'request_status' cannot be found in JSON response: {0}".format(request_response)) sys.exit(1) def get_request_id(json_response): @@ -684,7 +684,7 @@ def set_solr_security_management(infra_solr_props, accessor, enable = True): post_configuration = create_configs('infra-solr-security-json', security_props, 'Turn {0} security.json manaul management by migrationHelper.py'.format(turn_status)) apply_configs(config, accessor, post_configuration) else: - print "Configuration 'infra-solr-security-json/infra_solr_security_manually_managed' has already set to '{0}'".format(set_value) + print("Configuration 'infra-solr-security-json/infra_solr_security_manually_managed' has already set to '{0}'".format(set_value)) def set_solr_name_rules(infra_solr_props, accessor, add = False): """ @@ -696,22 +696,22 @@ def set_solr_name_rules(infra_solr_props, accessor, add = False): if 'content' in infra_solr_env_props and (name_rules_param not in infra_solr_env_props['content']) is add: if add: - print "Adding 'SOLR_KERB_NAME_RULES' to 'infra-solr-env/content'" + print("Adding 'SOLR_KERB_NAME_RULES' to 'infra-solr-env/content'") new_content = insert_string_before(infra_solr_env_props['content'], "SOLR_KERB_KEYTAB", name_rules_param) infra_solr_env_props['content'] = new_content post_configuration = create_configs('infra-solr-env', infra_solr_env_props, 'Add "SOLR_KERB_NAME_RULES" by migrationHelper.py') apply_configs(config, accessor, post_configuration) else: - print "Removing 'SOLR_KERB_NAME_RULES' from 'infra-solr-env/content'" + print("Removing 'SOLR_KERB_NAME_RULES' from 'infra-solr-env/content'") new_content = infra_solr_env_props['content'].replace(name_rules_param, '') infra_solr_env_props['content'] = new_content post_configuration = create_configs('infra-solr-env', infra_solr_env_props, 'Remove "SOLR_KERB_NAME_RULES" by migrationHelper.py') apply_configs(config, accessor, post_configuration) else: if add: - print "'SOLR_KERB_NAME_RULES' has already set in configuration 'infra-solr-env/content'" + print("'SOLR_KERB_NAME_RULES' has already set in configuration 'infra-solr-env/content'") else: - print "Configuration 'infra-solr-env/content' does not contain 'SOLR_KERB_NAME_RULES'" + print("Configuration 'infra-solr-env/content' does not contain 'SOLR_KERB_NAME_RULES'") def apply_configs(config, accessor, post_configuration): cluster = config.get('ambari_server', 'cluster') @@ -805,9 +805,9 @@ def get_input_output_solr_url(src_solr_urls, target_solr_urls): output_solr_urls = target_solr_urls hostname = socket.getfqdn() if any(hostname in s for s in input_solr_urls): - input_solr_urls = filter(lambda x: hostname in x, input_solr_urls) + input_solr_urls = [x for x in input_solr_urls if hostname in x] if any(hostname in s for s in output_solr_urls): - output_solr_urls = filter(lambda x: hostname in x, output_solr_urls) + output_solr_urls = [x for x in output_solr_urls if hostname in x] common_url_list = intersect(input_solr_urls, output_solr_urls) if common_url_list: input_solr_urls = common_url_list @@ -876,13 +876,13 @@ def monitor_solr_async_request(options, config, status_request, request_id): if async_request_failed: if async_request_timed_out: - print async_request_timeout_msg + print(async_request_timeout_msg) sys.exit(1) else: - print async_request_fail_msg + print(async_request_fail_msg) sys.exit(1) else: - print async_request_success_msg + print(async_request_success_msg) return request_id @@ -898,7 +898,7 @@ def delete_collection(options, config, collection, solr_urls, response_data_map) raise Exception("{0} command failed: {1}".format(delete_collection_json_cmd, str(err))) response=json.loads(str(out)) if 'requestid' in response: - print 'Deleting collection {0} request sent. {1}DONE{2}'.format(collection, colors.OKGREEN, colors.ENDC) + print('Deleting collection {0} request sent. {1}DONE{2}'.format(collection, colors.OKGREEN, colors.ENDC)) response_data_map['request_id']=response['requestid'] response_data_map['status_request']=REQUEST_STATUS_SOLR_COLLECTION_URL.format(solr_url, response['requestid']) return collection @@ -915,7 +915,7 @@ def create_collection(options, config, solr_urls, collection, config_set, shards raise Exception("{0} command failed: {1}".format(create_collection_json_cmd, str(err))) response=json.loads(str(out)) if 'success' in response: - print 'Creating collection {0} was {1}SUCCESSFUL{2}'.format(collection, colors.OKGREEN, colors.ENDC) + print('Creating collection {0} was {1}SUCCESSFUL{2}'.format(collection, colors.OKGREEN, colors.ENDC)) return collection else: raise Exception("CREATE collection ('{0}') failed. ({1}) Response: {1}".format(collection, str(out))) @@ -930,7 +930,7 @@ def reload_collection(options, config, solr_urls, collection): raise Exception("{0} command failed: {1}".format(reload_collection_json_cmd, str(err))) response=json.loads(str(out)) if 'success' in response: - print 'Reloading collection {0} was {1}SUCCESSFUL{2}'.format(collection, colors.OKGREEN, colors.ENDC) + print('Reloading collection {0} was {1}SUCCESSFUL{2}'.format(collection, colors.OKGREEN, colors.ENDC)) return collection else: raise Exception("RELOAD collection ('{0}') failed. ({1}) Response: {1}".format(collection, str(out))) @@ -1018,9 +1018,9 @@ def list_collections(options, config, output_file, include_number_of_docs=False) if output_file.endswith(dump_json_file): skip_dump=True if skip_dump: - print 'Skipping collection dump file generation: {0}'.format(output_file) + print('Skipping collection dump file generation: {0}'.format(output_file)) if not os.path.exists(output_file): - print "{0}FAIL{1}: Collection dump file '{2}' does not exist.".format(colors.FAIL, colors.ENDC, output_file) + print("{0}FAIL{1}: Collection dump file '{2}' does not exist.".format(colors.FAIL, colors.ENDC, output_file)) sys.exit(1) else: command_suffix = '--dump-collections --output {0}'.format(output_file) @@ -1040,7 +1040,7 @@ def list_collections(options, config, output_file, include_number_of_docs=False) sys.stdout.flush() logger.debug(str(out)) collections_data = get_collections_data(output_file) - return collections_data.keys() if collections_data is not None else [] + return list(collections_data.keys()) if collections_data is not None else [] def get_collections_data(output_file): return read_json(output_file) @@ -1059,7 +1059,7 @@ def delete_logsearch_collections(options, config, collections_json_location, col retry(monitor_solr_async_request, options, config, response_map['status_request'], response_map['request_id'], context="[Monitor Solr async request, id: {0}]".format(response_map['request_id'])) else: - print 'Collection {0} does not exist or filtered out. Skipping delete operation.'.format(service_logs_collection) + print('Collection {0} does not exist or filtered out. Skipping delete operation.'.format(service_logs_collection)) if audit_logs_collection in collections: solr_urls = get_solr_urls(options, config, audit_logs_collection, collections_json_location) response_map={} @@ -1067,7 +1067,7 @@ def delete_logsearch_collections(options, config, collections_json_location, col retry(monitor_solr_async_request, options, config, response_map['status_request'], response_map['request_id'], context="[Monitor Solr async request, id: {0}]".format(response_map['request_id'])) else: - print 'Collection {0} does not exist or filtered out. Skipping delete operation.'.format(audit_logs_collection) + print('Collection {0} does not exist or filtered out. Skipping delete operation.'.format(audit_logs_collection)) if history_collection in collections: solr_urls = get_solr_urls(options, config, history_collection, collections_json_location) response_map={} @@ -1075,7 +1075,7 @@ def delete_logsearch_collections(options, config, collections_json_location, col retry(monitor_solr_async_request, options, config, response_map['status_request'], response_map['request_id'], context="[Monitor Solr async request, id: {0}]".format(response_map['request_id'])) else: - print 'Collection {0} does not exist or filtered out. Skipping delete operation.'.format(history_collection) + print('Collection {0} does not exist or filtered out. Skipping delete operation.'.format(history_collection)) def delete_atlas_collections(options, config, collections_json_location, collections): fulltext_collection = config.get('atlas_collections', 'fulltext_index_name') @@ -1088,7 +1088,7 @@ def delete_atlas_collections(options, config, collections_json_location, collect retry(monitor_solr_async_request, options, config, response_map['status_request'], response_map['request_id'], context="[Monitor Solr async request, id: {0}]".format(response_map['request_id'])) else: - print 'Collection {0} does not exist or filtered out. Skipping delete operation.'.format(fulltext_collection) + print('Collection {0} does not exist or filtered out. Skipping delete operation.'.format(fulltext_collection)) if edge_index_collection in collections: solr_urls = get_solr_urls(options, config, edge_index_collection, collections_json_location) response_map={} @@ -1096,7 +1096,7 @@ def delete_atlas_collections(options, config, collections_json_location, collect retry(monitor_solr_async_request, options, config, response_map['status_request'], response_map['request_id'], context="[Monitor Solr async request, id: {0}]".format(response_map['request_id'])) else: - print 'Collection {0} does not exist or filtered out. Skipping delete operation.'.format(edge_index_collection) + print('Collection {0} does not exist or filtered out. Skipping delete operation.'.format(edge_index_collection)) if vertex_index_collection in collections: solr_urls = get_solr_urls(options, config, vertex_index_collection, collections_json_location) response_map={} @@ -1104,7 +1104,7 @@ def delete_atlas_collections(options, config, collections_json_location, collect retry(monitor_solr_async_request, options, config, response_map['status_request'], response_map['request_id'], context="[Monitor Solr async request, id: {0}]".format(response_map['request_id'])) else: - print 'Collection {0} does not exist or filtered out. Skipping delete operation.'.format(vertex_index_collection) + print('Collection {0} does not exist or filtered out. Skipping delete operation.'.format(vertex_index_collection)) def delete_ranger_collection(options, config, collections_json_location, collections): ranger_collection_name = config.get('ranger_collection', 'ranger_collection_name') @@ -1115,7 +1115,7 @@ def delete_ranger_collection(options, config, collections_json_location, collect retry(monitor_solr_async_request, options, config, response_map['status_request'], response_map['request_id'], context="[Monitor Solr async request, id: {0}]".format(response_map['request_id'])) else: - print 'Collection {0} does not exist or filtered out. Skipping delete operation'.format(ranger_collection_name) + print('Collection {0} does not exist or filtered out. Skipping delete operation'.format(ranger_collection_name)) def delete_collections(options, config, service_filter): collections_json_location = COLLECTIONS_DATA_JSON_LOCATION.format("delete_collections.json") @@ -1162,30 +1162,30 @@ def evaluate_check_shard_result(collection, result, skip_index_size = False): active_shards = result['active_shards'] all_shards = result['all_shards'] warnings = 0 - print 30 * "-" - print "Number of shards: {0}".format(str(len(all_shards))) + print(30 * "-") + print("Number of shards: {0}".format(str(len(all_shards)))) for shard in all_shards: if shard in active_shards: - print "{0}OK{1}: Found active leader replica for {2}" \ - .format(colors.OKGREEN, colors.ENDC, shard) + print("{0}OK{1}: Found active leader replica for {2}" \ + .format(colors.OKGREEN, colors.ENDC, shard)) else: warnings=warnings+1 - print "{0}WARNING{1}: Not found any active leader replicas for {2}, migration will probably fail, fix or delete the shard if it is possible." \ - .format(colors.WARNING, colors.ENDC, shard) + print("{0}WARNING{1}: Not found any active leader replicas for {2}, migration will probably fail, fix or delete the shard if it is possible." \ + .format(colors.WARNING, colors.ENDC, shard)) if not skip_index_size: index_size_map = result['index_size_map'] host_index_size_map = result['host_index_size_map'] if index_size_map: - print "Index size per shard for {0}:".format(collection) + print("Index size per shard for {0}:".format(collection)) for shard in index_size_map: - print " - {0}: {1}".format(shard, human_size(index_size_map[shard])) + print(" - {0}: {1}".format(shard, human_size(index_size_map[shard]))) if host_index_size_map: - print "Index size per host for {0} (consider this for backup): ".format(collection) + print("Index size per host for {0} (consider this for backup): ".format(collection)) for host in host_index_size_map: - print " - {0}: {1}".format(host, human_size(host_index_size_map[host])) + print(" - {0}: {1}".format(host, human_size(host_index_size_map[host]))) evaluate_result['host_index_size_map'] = host_index_size_map - print 30 * "-" + print(30 * "-") evaluate_result['warnings'] = warnings return evaluate_result @@ -1196,7 +1196,7 @@ def check_shard_for_collection(config, collection, skip_index_size = False): index_size_map = {} host_index_size_map = {} collections_data = get_collections_data(COLLECTIONS_DATA_JSON_LOCATION.format("check_collections.json")) - print "Checking available shards for '{0}' collection...".format(collection) + print("Checking available shards for '{0}' collection...".format(collection)) if collection in collections_data: collection_details = collections_data[collection] if 'shards' in collection_details: @@ -1275,7 +1275,7 @@ def is_collection_empty(docs_map, collection): if collection in docs_map: num_docs=docs_map[collection] if num_docs == -1: - print "Number of documents: -1. That means the number of docs was not provided in the collection dump." + print("Number of documents: -1. That means the number of docs was not provided in the collection dump.") elif num_docs == 0: result = True return result @@ -1337,7 +1337,7 @@ def update_state_json(original_collection, collection, config, options): ulog_dir = core_data['ulogDir'] if 'ulogDir' in core_data else None if cores_to_skip and (core in cores_to_skip or (core in core_pairs and core_pairs[core] in cores_to_skip)): - print "Skipping core '{0}' as it is in skip-cores list (or its original pair: '{1}')".format(core, core_pairs[core]) + print("Skipping core '{0}' as it is in skip-cores list (or its original pair: '{1}')".format(core, core_pairs[core])) elif replica in core_details: old_core_node=core_details[replica]['core_node'] new_core_node=core_details[replica]['new_core_node'] @@ -1382,16 +1382,16 @@ def do_backup_request(options, accessor, parser, config, collection, index_locat request_id = get_request_id(response) sys.stdout.write(colors.OKGREEN + 'DONE\n' + colors.ENDC) sys.stdout.flush() - print 'Backup command request id: {0}'.format(request_id) - if options.async: - print "Backup {0} collection request sent to Ambari server. Check Ambari UI about the results.".format(collection) + print('Backup command request id: {0}'.format(request_id)) + if options.async_op: + print("Backup {0} collection request sent to Ambari server. Check Ambari UI about the results.".format(collection)) sys.exit(0) else: sys.stdout.write("Start monitoring Ambari request with id {0} ...".format(request_id)) sys.stdout.flush() cluster = config.get('ambari_server', 'cluster') monitor_request(options, accessor, cluster, request_id, 'Backup Solr collection: ' + collection) - print "Backup collection '{0}'... {1}DONE{2}".format(collection, colors.OKGREEN, colors.ENDC) + print("Backup collection '{0}'... {1}DONE{2}".format(collection, colors.OKGREEN, colors.ENDC)) def do_migrate_request(options, accessor, parser, config, collection, index_location): sys.stdout.write("Sending migrate collection request ('{0}') to Ambari to process (migrate folder: '{1}')..." @@ -1401,16 +1401,16 @@ def do_migrate_request(options, accessor, parser, config, collection, index_loca request_id = get_request_id(response) sys.stdout.write(colors.OKGREEN + 'DONE\n' + colors.ENDC) sys.stdout.flush() - print 'Migrate command request id: {0}'.format(request_id) - if options.async: - print "Migrate {0} collection index request sent to Ambari server. Check Ambari UI about the results.".format(collection) + print('Migrate command request id: {0}'.format(request_id)) + if options.async_op: + print("Migrate {0} collection index request sent to Ambari server. Check Ambari UI about the results.".format(collection)) sys.exit(0) else: sys.stdout.write("Start monitoring Ambari request with id {0} ...".format(request_id)) sys.stdout.flush() cluster = config.get('ambari_server', 'cluster') monitor_request(options, accessor, cluster, request_id, 'Migrate Solr collection index: ' + collection) - print "Migrate index '{0}'... {1}DONE{2}".format(collection, colors.OKGREEN, colors.ENDC) + print("Migrate index '{0}'... {1}DONE{2}".format(collection, colors.OKGREEN, colors.ENDC)) def do_restore_request(options, accessor, parser, config, original_collection, collection, config_set, index_location, shards, hdfs_path): sys.stdout.write("Sending restore collection request ('{0}') to Ambari to process (backup location: '{1}')..." @@ -1420,16 +1420,16 @@ def do_restore_request(options, accessor, parser, config, original_collection, c request_id = get_request_id(response) sys.stdout.write(colors.OKGREEN + 'DONE\n' + colors.ENDC) sys.stdout.flush() - print 'Restore command request id: {0}'.format(request_id) - if options.async: - print "Restore {0} collection request sent to Ambari server. Check Ambari UI about the results.".format(collection) + print('Restore command request id: {0}'.format(request_id)) + if options.async_op: + print("Restore {0} collection request sent to Ambari server. Check Ambari UI about the results.".format(collection)) sys.exit(0) else: sys.stdout.write("Start monitoring Ambari request with id {0} ...".format(request_id)) sys.stdout.flush() cluster = config.get('ambari_server', 'cluster') monitor_request(options, accessor, cluster, request_id, 'Restore Solr collection: ' + collection) - print "Restoring collection '{0}'... {1}DONE{2}".format(collection, colors.OKGREEN, colors.ENDC) + print("Restoring collection '{0}'... {1}DONE{2}".format(collection, colors.OKGREEN, colors.ENDC)) def get_ranger_index_location(collection, config, options): ranger_index_location = None @@ -1440,8 +1440,8 @@ def get_ranger_index_location(collection, config, options): elif config.has_option('ranger_collection', 'backup_path'): ranger_index_location = config.get('ranger_collection', 'backup_path') else: - print "'backup_path'is missing from config file and --index-location or --ranger-index-location options are missing as well. Backup collection {0} {1}FAILED{2}." \ - .format(collection, colors.FAIL, colors.ENDC) + print("'backup_path'is missing from config file and --index-location or --ranger-index-location options are missing as well. Backup collection {0} {1}FAILED{2}." \ + .format(collection, colors.FAIL, colors.ENDC)) sys.exit(1) return ranger_index_location @@ -1454,8 +1454,8 @@ def get_atlas_index_location(collection, config, options): elif config.has_option('atlas_collections', 'backup_path'): atlas_index_location = os.path.join(config.get('atlas_collections', 'backup_path'), collection) else: - print "'backup_path'is missing from config file and --index-location or --atlas-index-location options are missing as well. Backup collection {0} {1}FAILED{2}." \ - .format(collection, colors.FAIL, colors.ENDC) + print("'backup_path'is missing from config file and --index-location or --atlas-index-location options are missing as well. Backup collection {0} {1}FAILED{2}." \ + .format(collection, colors.FAIL, colors.ENDC)) sys.exit(1) return atlas_index_location @@ -1467,40 +1467,40 @@ def backup_collections(options, accessor, parser, config, service_filter): collection_name = config.get('ranger_collection', 'ranger_collection_name') if collection_name in collections: if is_collection_empty(num_docs_map, collection_name): - print "Collection '{0}' is empty. Backup is not required.".format(collection_name) + print("Collection '{0}' is empty. Backup is not required.".format(collection_name)) else: ranger_index_location=get_ranger_index_location(collection_name, config, options) do_backup_request(options, accessor, parser, config, collection_name, ranger_index_location) else: - print 'Collection {0} does not exist or filtered out. Skipping backup operation.'.format(collection_name) + print('Collection {0} does not exist or filtered out. Skipping backup operation.'.format(collection_name)) if is_atlas_available(config, service_filter): fulltext_index_collection = config.get('atlas_collections', 'fulltext_index_name') if fulltext_index_collection in collections: if is_collection_empty(num_docs_map, fulltext_index_collection): - print "Collection '{0}' is empty. Backup is not required.".format(fulltext_index_collection) + print("Collection '{0}' is empty. Backup is not required.".format(fulltext_index_collection)) else: fulltext_index_location = get_atlas_index_location(fulltext_index_collection, config, options) do_backup_request(options, accessor, parser, config, fulltext_index_collection, fulltext_index_location) else: - print 'Collection {0} does not exist or filtered out. Skipping backup operation.'.format(fulltext_index_collection) + print('Collection {0} does not exist or filtered out. Skipping backup operation.'.format(fulltext_index_collection)) vertex_index_collection = config.get('atlas_collections', 'vertex_index_name') if vertex_index_collection in collections: if is_collection_empty(num_docs_map, vertex_index_collection): - print "Collection '{0}' is empty. Backup is not required.".format(vertex_index_collection) + print("Collection '{0}' is empty. Backup is not required.".format(vertex_index_collection)) else: vertex_index_location = get_atlas_index_location(vertex_index_collection, config, options) do_backup_request(options, accessor, parser, config, vertex_index_collection, vertex_index_location) else: - print 'Collection {0} does not exist or filtered out. Skipping backup operation.'.format(vertex_index_collection) + print('Collection {0} does not exist or filtered out. Skipping backup operation.'.format(vertex_index_collection)) edge_index_collection = config.get('atlas_collections', 'edge_index_name') if edge_index_collection in collections: if is_collection_empty(num_docs_map, edge_index_collection): - print "Collection '{0}' is empty. Backup is not required.".format(edge_index_collection) + print("Collection '{0}' is empty. Backup is not required.".format(edge_index_collection)) else: edge_index_location = get_atlas_index_location(edge_index_collection, config, options) do_backup_request(options, accessor, parser, config, edge_index_collection, edge_index_location) else: - print 'Collection {0} does not exist or filtered out. Skipping backup operation.'.format(edge_index_collection) + print('Collection {0} does not exist or filtered out. Skipping backup operation.'.format(edge_index_collection)) def migrate_snapshots(options, accessor, parser, config, service_filter): if is_ranger_available(config, service_filter): @@ -1509,26 +1509,26 @@ def migrate_snapshots(options, accessor, parser, config, service_filter): ranger_index_location=get_ranger_index_location(collection_name, config, options) do_migrate_request(options, accessor, parser, config, collection_name, ranger_index_location) else: - print "Collection '{0}' backup index has filtered out. Skipping migrate operation.".format(collection_name) + print("Collection '{0}' backup index has filtered out. Skipping migrate operation.".format(collection_name)) if is_atlas_available(config, service_filter): fulltext_index_collection = config.get('atlas_collections', 'fulltext_index_name') if options.collection is None or options.collection == fulltext_index_collection: fulltext_index_location=get_atlas_index_location(fulltext_index_collection, config, options) do_migrate_request(options, accessor, parser, config, fulltext_index_collection, fulltext_index_location) else: - print "Collection '{0}' backup index has filtered out. Skipping migrate operation.".format(fulltext_index_collection) + print("Collection '{0}' backup index has filtered out. Skipping migrate operation.".format(fulltext_index_collection)) vertex_index_collection = config.get('atlas_collections', 'vertex_index_name') if options.collection is None or options.collection == vertex_index_collection: vertex_index_location=get_atlas_index_location(vertex_index_collection, config, options) do_migrate_request(options, accessor, parser, config, vertex_index_collection, vertex_index_location) else: - print "Collection '{0}' backup index has filtered out. Skipping migrate operation.".format(vertex_index_collection) + print("Collection '{0}' backup index has filtered out. Skipping migrate operation.".format(vertex_index_collection)) edge_index_collection = config.get('atlas_collections', 'edge_index_name') if options.collection is None or options.collection == edge_index_collection: edge_index_location=get_atlas_index_location(edge_index_collection, config, options) do_migrate_request(options, accessor, parser, config, edge_index_collection, edge_index_location) else: - print "Collection '{0}' backup index has filtered out. Skipping migrate operation.".format(edge_index_collection) + print("Collection '{0}' backup index has filtered out. Skipping migrate operation.".format(edge_index_collection)) def create_backup_collections(options, accessor, parser, config, service_filter): collections_json_location = COLLECTIONS_DATA_JSON_LOCATION.format("before_restore_collections.json") @@ -1540,11 +1540,11 @@ def create_backup_collections(options, accessor, parser, config, service_filter) backup_ranger_collection = config.get('ranger_collection', 'backup_ranger_collection_name') if original_ranger_collection in collections: if is_collection_empty(num_docs_map, original_ranger_collection): - print "Collection '{0}' was empty during backup. It won't need a backup collection.".format(original_ranger_collection) + print("Collection '{0}' was empty during backup. It won't need a backup collection.".format(original_ranger_collection)) else: if backup_ranger_collection not in collections: if options.collection is not None and options.collection != backup_ranger_collection: - print "Collection {0} has filtered out. Skipping create operation.".format(backup_ranger_collection) + print("Collection {0} has filtered out. Skipping create operation.".format(backup_ranger_collection)) else: solr_urls = get_solr_urls(options, config, backup_ranger_collection, collections_json_location) backup_ranger_config_set = config.get('ranger_collection', 'backup_ranger_config_set_name') @@ -1553,18 +1553,18 @@ def create_backup_collections(options, accessor, parser, config, service_filter) retry(create_collection, options, config, solr_urls, backup_ranger_collection, backup_ranger_config_set, backup_ranger_shards, replica_number, backup_ranger_max_shards, context="[Create Solr Collections]") else: - print "Collection {0} has already exist. Skipping create operation.".format(backup_ranger_collection) + print("Collection {0} has already exist. Skipping create operation.".format(backup_ranger_collection)) if is_atlas_available(config, service_filter): backup_atlas_config_set = config.get('atlas_collections', 'config_set') backup_fulltext_index_name = config.get('atlas_collections', 'backup_fulltext_index_name') original_fulltext_index_name = config.get('atlas_collections', 'fulltext_index_name') if original_fulltext_index_name in collections: if is_collection_empty(num_docs_map, original_fulltext_index_name): - print "Collection '{0}' was empty during backup. It won't need a backup collection.".format(original_fulltext_index_name) + print("Collection '{0}' was empty during backup. It won't need a backup collection.".format(original_fulltext_index_name)) else: if backup_fulltext_index_name not in collections: if options.collection is not None and options.collection != backup_fulltext_index_name: - print "Collection {0} has filtered out. Skipping create operation.".format(backup_fulltext_index_name) + print("Collection {0} has filtered out. Skipping create operation.".format(backup_fulltext_index_name)) else: solr_urls = get_solr_urls(options, config, backup_fulltext_index_name, collections_json_location) backup_fulltext_index_shards = config.get('atlas_collections', 'fulltext_index_shards') @@ -1572,17 +1572,17 @@ def create_backup_collections(options, accessor, parser, config, service_filter) retry(create_collection, options, config, solr_urls, backup_fulltext_index_name, backup_atlas_config_set, backup_fulltext_index_shards, replica_number, backup_fulltext_index_max_shards, context="[Create Solr Collections]") else: - print "Collection {0} has already exist. Skipping create operation.".format(backup_fulltext_index_name) + print("Collection {0} has already exist. Skipping create operation.".format(backup_fulltext_index_name)) backup_edge_index_name = config.get('atlas_collections', 'backup_edge_index_name') original_edge_index_name = config.get('atlas_collections', 'edge_index_name') if original_edge_index_name in collections: if is_collection_empty(num_docs_map, original_edge_index_name): - print "Collection '{0}' was empty during backup. It won't need a backup collection.".format(original_edge_index_name) + print("Collection '{0}' was empty during backup. It won't need a backup collection.".format(original_edge_index_name)) else: if backup_edge_index_name not in collections: if options.collection is not None and options.collection != backup_edge_index_name: - print "Collection {0} has filtered out. Skipping create operation.".format(backup_edge_index_name) + print("Collection {0} has filtered out. Skipping create operation.".format(backup_edge_index_name)) else: solr_urls = get_solr_urls(options, config, backup_edge_index_name, collections_json_location) backup_edge_index_shards = config.get('atlas_collections', 'edge_index_shards') @@ -1590,17 +1590,17 @@ def create_backup_collections(options, accessor, parser, config, service_filter) retry(create_collection, options, config, solr_urls, backup_edge_index_name, backup_atlas_config_set, backup_edge_index_shards, replica_number, backup_edge_index_max_shards, context="[Create Solr Collections]") else: - print "Collection {0} has already exist. Skipping create operation.".format(backup_edge_index_name) + print("Collection {0} has already exist. Skipping create operation.".format(backup_edge_index_name)) backup_vertex_index_name = config.get('atlas_collections', 'backup_vertex_index_name') original_vertex_index_name = config.get('atlas_collections', 'vertex_index_name') if original_vertex_index_name in collections: if is_collection_empty(num_docs_map, original_vertex_index_name): - print "Collection '{0}' was empty during backup. It won't need a backup collection.".format(original_vertex_index_name) + print("Collection '{0}' was empty during backup. It won't need a backup collection.".format(original_vertex_index_name)) else: if backup_vertex_index_name not in collections: if options.collection is not None and options.collection != backup_vertex_index_name: - print "Collection {0} has filtered out. Skipping create operation.".format(backup_vertex_index_name) + print("Collection {0} has filtered out. Skipping create operation.".format(backup_vertex_index_name)) else: solr_urls = get_solr_urls(options, config, backup_vertex_index_name, collections_json_location) backup_vertex_index_shards = config.get('atlas_collections', 'vertex_index_shards') @@ -1608,7 +1608,7 @@ def create_backup_collections(options, accessor, parser, config, service_filter) retry(create_collection, options, config, solr_urls, backup_vertex_index_name, backup_atlas_config_set, backup_vertex_index_shards, replica_number, backup_vertex_index_max_shards, context="[Create Solr Collections]") else: - print "Collection {0} has already exist. Skipping create operation.".format(backup_fulltext_index_name) + print("Collection {0} has already exist. Skipping create operation.".format(backup_fulltext_index_name)) def restore_collections(options, accessor, parser, config, service_filter): collections=list_collections(options, config, COLLECTIONS_DATA_JSON_LOCATION.format("restore_collections.json")) @@ -1631,7 +1631,7 @@ def restore_collections(options, accessor, parser, config, service_filter): ranger_index_location=get_ranger_index_location(collection_name, config, options) do_restore_request(options, accessor, parser, config, collection_name, backup_ranger_collection, backup_ranger_config_set_name, ranger_index_location, backup_ranger_shards, hdfs_base_path) else: - print "Collection '{0}' does not exist or filtered out. Skipping restore operation.".format(backup_ranger_collection) + print("Collection '{0}' does not exist or filtered out. Skipping restore operation.".format(backup_ranger_collection)) if is_atlas_available(config, service_filter): hdfs_base_path = None @@ -1650,7 +1650,7 @@ def restore_collections(options, accessor, parser, config, service_filter): fulltext_index_location=get_atlas_index_location(fulltext_index_collection, config, options) do_restore_request(options, accessor, parser, config, fulltext_index_collection, backup_fulltext_index_name, atlas_config_set, fulltext_index_location, backup_fulltext_index_shards, hdfs_base_path) else: - print "Collection '{0}' does not exist or filtered out. Skipping restore operation.".format(fulltext_index_collection) + print("Collection '{0}' does not exist or filtered out. Skipping restore operation.".format(fulltext_index_collection)) edge_index_collection = config.get('atlas_collections', 'edge_index_name') backup_edge_index_name = config.get('atlas_collections', 'backup_edge_index_name') @@ -1659,7 +1659,7 @@ def restore_collections(options, accessor, parser, config, service_filter): edge_index_location=get_atlas_index_location(edge_index_collection, config, options) do_restore_request(options, accessor, parser, config, edge_index_collection, backup_edge_index_name, atlas_config_set, edge_index_location, backup_edge_index_shards, hdfs_base_path) else: - print "Collection '{0}' does not exist or filtered out. Skipping restore operation.".format(edge_index_collection) + print("Collection '{0}' does not exist or filtered out. Skipping restore operation.".format(edge_index_collection)) vertex_index_collection = config.get('atlas_collections', 'vertex_index_name') backup_vertex_index_name = config.get('atlas_collections', 'backup_vertex_index_name') @@ -1668,7 +1668,7 @@ def restore_collections(options, accessor, parser, config, service_filter): vertex_index_location=get_atlas_index_location(vertex_index_collection, config, options) do_restore_request(options, accessor, parser, config, vertex_index_collection, backup_vertex_index_name, atlas_config_set, vertex_index_location, backup_vertex_index_shards, hdfs_base_path) else: - print "Collection '{0}' does not exist or filtered out. Skipping restore operation.".format(vertex_index_collection) + print("Collection '{0}' does not exist or filtered out. Skipping restore operation.".format(vertex_index_collection)) def reload_collections(options, accessor, parser, config, service_filter): collections_json_location = COLLECTIONS_DATA_JSON_LOCATION.format("reload_collections.json") @@ -1680,35 +1680,35 @@ def reload_collections(options, accessor, parser, config, service_filter): solr_urls = get_solr_urls(options, config, backup_ranger_collection, collections_json_location) retry(reload_collection, options, config, solr_urls, backup_ranger_collection, context="[Reload Solr Collections]") else: - print "Collection '{0}' does not exist or filtered out. Skipping reload operation.".format(backup_ranger_collection) + print("Collection '{0}' does not exist or filtered out. Skipping reload operation.".format(backup_ranger_collection)) if is_atlas_available(config, service_filter): backup_fulltext_index_name = config.get('atlas_collections', 'backup_fulltext_index_name') if backup_fulltext_index_name in collections: solr_urls = get_solr_urls(options, config, backup_fulltext_index_name, collections_json_location) retry(reload_collection, options, config, solr_urls, backup_fulltext_index_name, context="[Reload Solr Collections]") else: - print "Collection '{0}' does not exist or filtered out. Skipping reload operation.".format(backup_fulltext_index_name) + print("Collection '{0}' does not exist or filtered out. Skipping reload operation.".format(backup_fulltext_index_name)) backup_edge_index_name = config.get('atlas_collections', 'backup_edge_index_name') if backup_edge_index_name in collections: solr_urls = get_solr_urls(options, config, backup_edge_index_name, collections_json_location) retry(reload_collection, options, config, solr_urls, backup_edge_index_name, context="[Reload Solr Collections]") else: - print "Collection '{0}' does not exist or filtered out. Skipping reload operation.".format(backup_edge_index_name) + print("Collection '{0}' does not exist or filtered out. Skipping reload operation.".format(backup_edge_index_name)) backup_vertex_index_name = config.get('atlas_collections', 'backup_vertex_index_name') if backup_vertex_index_name in collections: solr_urls = get_solr_urls(options, config, backup_vertex_index_name, collections_json_location) retry(reload_collection, options, config, solr_urls, backup_vertex_index_name, context="[Reload Solr Collections]") else: - print "Collection '{0}' does not exist or filtered out. Skipping reload operation.".format(backup_fulltext_index_name) + print("Collection '{0}' does not exist or filtered out. Skipping reload operation.".format(backup_fulltext_index_name)) def validate_ini_file(options, parser): if options.ini_file is None: parser.print_help() - print 'ini-file option is missing' + print('ini-file option is missing') sys.exit(1) elif not os.path.isfile(options.ini_file): parser.print_help() - print 'ini file ({0}) does not exist'.format(options.ini_file) + print('ini file ({0}) does not exist'.format(options.ini_file)) sys.exit(1) def rolling_restart(options, accessor, parser, config, service_name, component_name, context): @@ -1718,7 +1718,7 @@ def rolling_restart(options, accessor, parser, config, service_name, component_n fault_tolerance = options.batch_fault_tolerance request_body = create_batch_command("RESTART", component_hosts, cluster, service_name, component_name, interval_secs, fault_tolerance, "Rolling restart Infra Solr Instances") post_json(accessor, BATCH_REQUEST_API_URL.format(cluster), request_body) - print "{0} request sent. (check Ambari UI about the requests)".format(context) + print("{0} request sent. (check Ambari UI about the requests)".format(context)) def update_state_jsons(options, accessor, parser, config, service_filter): collections=list_collections(options, config, COLLECTIONS_DATA_JSON_LOCATION.format("collections.json")) @@ -1729,26 +1729,26 @@ def update_state_jsons(options, accessor, parser, config, service_filter): if backup_ranger_collection in collections: update_state_json(original_ranger_collection, backup_ranger_collection, config, options) else: - print "Collection '{0}' does not exist or filtered out. Skipping update collection state operation.".format(backup_ranger_collection) + print("Collection '{0}' does not exist or filtered out. Skipping update collection state operation.".format(backup_ranger_collection)) if is_atlas_available(config, service_filter): original_fulltext_index_name = config.get('atlas_collections', 'fulltext_index_name') backup_fulltext_index_name = config.get('atlas_collections', 'backup_fulltext_index_name') if backup_fulltext_index_name in collections: update_state_json(original_fulltext_index_name, backup_fulltext_index_name, config, options) else: - print "Collection '{0}' does not exist or filtered out. Skipping update collection state operation.".format(backup_fulltext_index_name) + print("Collection '{0}' does not exist or filtered out. Skipping update collection state operation.".format(backup_fulltext_index_name)) original_edge_index_name = config.get('atlas_collections', 'edge_index_name') backup_edge_index_name = config.get('atlas_collections', 'backup_edge_index_name') if backup_edge_index_name in collections: update_state_json(original_edge_index_name, backup_edge_index_name, config, options) else: - print "Collection '{0}' does not exist or filtered out. Skipping update collection state operation.".format(backup_edge_index_name) + print("Collection '{0}' does not exist or filtered out. Skipping update collection state operation.".format(backup_edge_index_name)) original_vertex_index_name = config.get('atlas_collections', 'vertex_index_name') backup_vertex_index_name = config.get('atlas_collections', 'backup_vertex_index_name') if backup_vertex_index_name in collections: update_state_json(original_vertex_index_name, backup_vertex_index_name, config, options) else: - print "Collection '{0}' does not exist or filtered out. Skipping update collection state operation.".format(backup_fulltext_index_name) + print("Collection '{0}' does not exist or filtered out. Skipping update collection state operation.".format(backup_fulltext_index_name)) def set_solr_authorization(options, accessor, parser, config, enable_authorization, fix_kerberos_config = False): solr_znode='/infra-solr' @@ -1760,12 +1760,12 @@ def set_solr_authorization(options, accessor, parser, config, enable_authorizati if kerberos_enabled == 'true': infra_solr_props = get_infra_solr_props(config, accessor) if enable_authorization: - print "Enable Solr security.json management by Ambari ... " + print("Enable Solr security.json management by Ambari ... ") set_solr_security_management(infra_solr_props, accessor, enable = False) if fix_kerberos_config: set_solr_name_rules(infra_solr_props, accessor, False) else: - print "Disable Solr authorization by uploading a new security.json and turn on security.json management by Ambari..." + print("Disable Solr authorization by uploading a new security.json and turn on security.json management by Ambari...") set_solr_security_management(infra_solr_props, accessor, enable = True) copy_znode(options, config, COLLECTIONS_DATA_JSON_LOCATION.format("security-without-authr.json"), "{0}/security.json".format(solr_znode), copy_from_local=True) @@ -1773,9 +1773,9 @@ def set_solr_authorization(options, accessor, parser, config, enable_authorizati set_solr_name_rules(infra_solr_props, accessor, True) else: if fix_kerberos_config: - print "Security is not enabled. Skipping enable/disable Solr authorization + fix infra-solr-env kerberos config operation." + print("Security is not enabled. Skipping enable/disable Solr authorization + fix infra-solr-env kerberos config operation.") else: - print "Security is not enabled. Skipping enable/disable Solr authorization operation." + print("Security is not enabled. Skipping enable/disable Solr authorization operation.") def summarize_shard_check_result(check_results, skip_warnings = False, skip_index_size = False): warnings = 0 @@ -1792,16 +1792,16 @@ def summarize_shard_check_result(check_results, skip_warnings = False, skip_inde index_size_per_host[host]=host_index_size_map[host] pass if not skip_index_size and index_size_per_host: - print "Full index size per hosts: (consider this for backup)" + print("Full index size per hosts: (consider this for backup)") for host in index_size_per_host: - print " - {0}: {1}".format(host, human_size(index_size_per_host[host])) + print(" - {0}: {1}".format(host, human_size(index_size_per_host[host]))) - print "All warnings: {0}".format(warnings) + print("All warnings: {0}".format(warnings)) if warnings != 0 and not skip_warnings: - print "Check shards - {0}FAILED{1} (warnings: {2}, fix warnings or use --skip-warnings flag to PASS) ".format(colors.FAIL, colors.ENDC, warnings) + print("Check shards - {0}FAILED{1} (warnings: {2}, fix warnings or use --skip-warnings flag to PASS) ".format(colors.FAIL, colors.ENDC, warnings)) sys.exit(1) else: - print "Check shards - {0}PASSED{1}".format(colors.OKGREEN, colors.ENDC) + print("Check shards - {0}PASSED{1}".format(colors.OKGREEN, colors.ENDC)) def check_shards(options, accessor, parser, config, backup_shards = False): collections=list_collections(options, config, COLLECTIONS_DATA_JSON_LOCATION.format("check_collections.json")) @@ -1814,7 +1814,7 @@ def check_shards(options, accessor, parser, config, backup_shards = False): ranger_collection_details = check_shard_for_collection(config, ranger_collection, options.skip_index_size) check_results[ranger_collection]=evaluate_check_shard_result(ranger_collection, ranger_collection_details, options.skip_index_size) else: - print "Collection '{0}' does not exist or filtered out. Skipping check collection operation.".format(ranger_collection) + print("Collection '{0}' does not exist or filtered out. Skipping check collection operation.".format(ranger_collection)) if is_atlas_available(config, service_filter): fulltext_index_name = config.get('atlas_collections', 'backup_fulltext_index_name') if backup_shards \ else config.get('atlas_collections', 'fulltext_index_name') @@ -1822,32 +1822,32 @@ def check_shards(options, accessor, parser, config, backup_shards = False): fulltext_collection_details = check_shard_for_collection(config, fulltext_index_name, options.skip_index_size) check_results[fulltext_index_name]=evaluate_check_shard_result(fulltext_index_name, fulltext_collection_details, options.skip_index_size) else: - print "Collection '{0}' does not exist or filtered out. Skipping check collection operation.".format(fulltext_index_name) + print("Collection '{0}' does not exist or filtered out. Skipping check collection operation.".format(fulltext_index_name)) edge_index_name = config.get('atlas_collections', 'backup_edge_index_name') if backup_shards \ else config.get('atlas_collections', 'edge_index_name') if edge_index_name in collections: edge_collection_details = check_shard_for_collection(config, edge_index_name, options.skip_index_size) check_results[edge_index_name]=evaluate_check_shard_result(edge_index_name, edge_collection_details, options.skip_index_size) else: - print "Collection '{0}' does not exist or filtered out. Skipping check collection operation.".format(edge_index_name) + print("Collection '{0}' does not exist or filtered out. Skipping check collection operation.".format(edge_index_name)) vertex_index_name = config.get('atlas_collections', 'backup_vertex_index_name') if backup_shards \ else config.get('atlas_collections', 'vertex_index_name') if vertex_index_name in collections: vertex_collection_details = check_shard_for_collection(config, vertex_index_name, options.skip_index_size) check_results[vertex_index_name]=evaluate_check_shard_result(vertex_index_name, vertex_collection_details, options.skip_index_size) else: - print "Collection '{0}' does not exist or filtered out. Skipping check collection operation.".format(fulltext_index_name) + print("Collection '{0}' does not exist or filtered out. Skipping check collection operation.".format(fulltext_index_name)) summarize_shard_check_result(check_results, options.skip_warnings, options.skip_index_size) def check_docs(options, accessor, parser, config): collections=list_collections(options, config, COLLECTIONS_DATA_JSON_LOCATION.format("check_docs_collections.json"), include_number_of_docs=True) if collections: - print "Get the number of documents per collections ..." + print("Get the number of documents per collections ...") docs_map = get_number_of_docs_map(COLLECTIONS_DATA_JSON_LOCATION.format("check_docs_collections.json")) for collection_docs_data in docs_map: - print "Collection: '{0}' - Number of docs: {1}".format(collection_docs_data, docs_map[collection_docs_data]) + print("Collection: '{0}' - Number of docs: {1}".format(collection_docs_data, docs_map[collection_docs_data])) else: - print "Check number of documents - Not found any collections." + print("Check number of documents - Not found any collections.") def run_solr_data_manager_on_collection(options, config, collections, src_collection, target_collection, collections_json_location, num_docs, skip_date_usage = True): @@ -1855,15 +1855,15 @@ def run_solr_data_manager_on_collection(options, config, collections, src_collec source_solr_urls = get_solr_urls(options, config, src_collection, collections_json_location) target_solr_urls = get_solr_urls(options, config, target_collection, collections_json_location) if is_collection_empty(num_docs, src_collection): - print "Collection '{0}' is empty. Skipping transport data operation.".format(target_collection) + print("Collection '{0}' is empty. Skipping transport data operation.".format(target_collection)) else: src_solr_url, target_solr_url = get_input_output_solr_url(source_solr_urls, target_solr_urls) keytab, principal = get_keytab_and_principal(config) date_format = "%Y-%m-%dT%H:%M:%S.%fZ" d = datetime.now() + timedelta(days=365) end = d.strftime(date_format) - print "Running solrDataManager.py (solr input collection: {0}, solr output collection: {1})"\ - .format(src_collection, target_collection) + print("Running solrDataManager.py (solr input collection: {0}, solr output collection: {1})"\ + .format(src_collection, target_collection)) solr_data_manager.verbose = options.verbose solr_data_manager.set_log_level(True) solr_data_manager.save("archive", src_solr_url, src_collection, "evtTime", "id", end, @@ -1872,7 +1872,7 @@ def run_solr_data_manager_on_collection(options, config, collections, src_collec None, None, None, None, None, target_collection, target_solr_url, "_version_", skip_date_usage) else: - print "Collection '{0}' or {1} does not exist or filtered out. Skipping transport data operation.".format(target_collection, src_collection) + print("Collection '{0}' or {1} does not exist or filtered out. Skipping transport data operation.".format(target_collection, src_collection)) def transfer_old_data(options, accessor, parser, config): collections_json_location = COLLECTIONS_DATA_JSON_LOCATION.format("transport_collections.json") @@ -1913,7 +1913,7 @@ if __name__=="__main__": parser.add_option("-v", "--verbose", dest="verbose", action="store_true", help="use for verbose logging") parser.add_option("-s", "--service-filter", dest="service_filter", default=None, type="string", help="run commands only selected services (comma separated: LOGSEARCH,ATLAS,RANGER)") parser.add_option("-c", "--collection", dest="collection", default=None, type="string", help="selected collection to run an operation") - parser.add_option("--async", dest="async", action="store_true", default=False, help="async Ambari operations (backup | restore | migrate)") + parser.add_option("--async", dest="async_op", action="store_true", default=False, help="async Ambari operations (backup | restore | migrate)") parser.add_option("--index-location", dest="index_location", type="string", help="location of the index backups. add ranger/atlas prefix after the path. required only if no backup path in the ini file") parser.add_option("--atlas-index-location", dest="atlas_index_location", type="string", help="location of the index backups (for atlas). required only if no backup path in the ini file") parser.add_option("--ranger-index-location", dest="ranger_index_location", type="string", help="location of the index backups (for ranger). required only if no backup path in the ini file") @@ -1945,11 +1945,11 @@ if __name__=="__main__": set_log_level(options.verbose) if options.verbose: - print "Run command with args: {0}".format(str(sys.argv)) + print("Run command with args: {0}".format(str(sys.argv))) validate_ini_file(options, parser) - config = ConfigParser.RawConfigParser() + config = configparser.RawConfigParser() config.read(options.ini_file) command_start_time = time.time() @@ -1958,7 +1958,7 @@ if __name__=="__main__": if options.action is None: parser.print_help() - print 'action option is missing' + print('action option is missing') sys.exit(1) else: if config.has_section('ambari_server'): @@ -1973,8 +1973,8 @@ if __name__=="__main__": local_host = socket.getfqdn() solr_hosts = config.get('infra_solr', 'hosts') if solr_hosts and local_host not in solr_hosts.split(","): - print "{0}WARNING{1}: Host '{2}' is not found in Infra Solr hosts ({3}). Migration commands won't work from here." \ - .format(colors.WARNING, colors.ENDC, local_host, solr_hosts) + print("{0}WARNING{1}: Host '{2}' is not found in Infra Solr hosts ({3}). Migration commands won't work from here." \ + .format(colors.WARNING, colors.ENDC, local_host, solr_hosts)) if options.action.lower() == 'backup': backup_ranger_configs(options, config, service_filter) backup_collections(options, accessor, parser, config, service_filter) @@ -2010,18 +2010,18 @@ if __name__=="__main__": if is_logsearch_available(config, service_filter): upgrade_logsearch_portal(options, accessor, parser, config) else: - print "LOGSEARCH service has not found in the config or filtered out." + print("LOGSEARCH service has not found in the config or filtered out.") elif options.action.lower() == 'upgrade-logfeeders': if is_logsearch_available(config, service_filter): upgrade_logfeeders(options, accessor, parser, config) else: - print "LOGSEARCH service has not found in the config or filtered out." + print("LOGSEARCH service has not found in the config or filtered out.") elif options.action.lower() == 'stop-logsearch': if is_logsearch_available(config, service_filter): service_components_command(options, accessor, parser, config, LOGSEARCH_SERVICE_NAME, LOGSEARCH_SERVER_COMPONENT_NAME, "STOP", "Stop") service_components_command(options, accessor, parser, config, LOGSEARCH_SERVICE_NAME, LOGSEARCH_LOGFEEDER_COMPONENT_NAME, "STOP", "Stop") else: - print "LOGSEARCH service has not found in the config or filtered out." + print("LOGSEARCH service has not found in the config or filtered out.") elif options.action.lower() == 'restart-solr': service_components_command(options, accessor, parser, config, SOLR_SERVICE_NAME, SOLR_COMPONENT_NAME, "RESTART", "Restart") elif options.action.lower() == 'restart-logsearch': @@ -2029,27 +2029,27 @@ if __name__=="__main__": service_components_command(options, accessor, parser, config, LOGSEARCH_SERVICE_NAME, LOGSEARCH_SERVER_COMPONENT_NAME, "RESTART", "Restart") service_components_command(options, accessor, parser, config, LOGSEARCH_SERVICE_NAME, LOGSEARCH_LOGFEEDER_COMPONENT_NAME, "RESTART", "Restart") else: - print "LOGSEARCH service has not found in the config or filtered out." + print("LOGSEARCH service has not found in the config or filtered out.") elif options.action.lower() == 'restart-atlas': if is_atlas_available(config, service_filter): service_components_command(options, accessor, parser, config, ATLAS_SERVICE_NAME, ATLAS_SERVER_COMPONENT_NAME, "RESTART", "Restart") else: - print "ATLAS service has not found in the config or filtered out." + print("ATLAS service has not found in the config or filtered out.") elif options.action.lower() == 'restart-ranger': if is_ranger_available(config, service_filter): service_components_command(options, accessor, parser, config, RANGER_SERVICE_NAME, RANGER_ADMIN_COMPONENT_NAME, "RESTART", "Restart") else: - print "RANGER service has not found in the config or filtered out." + print("RANGER service has not found in the config or filtered out.") elif options.action.lower() == 'rolling-restart-ranger': if is_ranger_available(config, service_filter): rolling_restart(options, accessor, parser, config, RANGER_SERVICE_NAME, RANGER_ADMIN_COMPONENT_NAME, "Rolling Restart Ranger Admin Instances") else: - print "RANGER service has not found in the config or filtered out." + print("RANGER service has not found in the config or filtered out.") elif options.action.lower() == 'rolling-restart-atlas': if is_atlas_available(config, service_filter): rolling_restart(options, accessor, parser, config, ATLAS_SERVICE_NAME, ATLAS_SERVER_COMPONENT_NAME, "Rolling Restart Atlas Server Instances") else: - print "ATLAS service has not found in the config or filtered out." + print("ATLAS service has not found in the config or filtered out.") elif options.action.lower() == 'rolling-restart-solr': rolling_restart(options, accessor, parser, config, SOLR_SERVICE_NAME, SOLR_COMPONENT_NAME, "Rolling Restart Infra Solr Instances") elif options.action.lower() == 'enable-solr-authorization': @@ -2072,15 +2072,15 @@ if __name__=="__main__": check_docs(options, accessor, parser, config) else: parser.print_help() - print 'action option is invalid (available actions: delete-collections | backup | cleanup-znodes | backup-and-cleanup | migrate | restore |' \ + print('action option is invalid (available actions: delete-collections | backup | cleanup-znodes | backup-and-cleanup | migrate | restore |' \ ' rolling-restart-solr | rolling-restart-ranger | rolling-restart-atlas | check-shards | check-backup-shards | check-docs | enable-solr-authorization |'\ ' disable-solr-authorization | fix-solr5-kerberos-config | fix-solr7-kerberos-config | upgrade-solr-clients | upgrade-solr-instances | upgrade-logsearch-portal |' \ ' upgrade-logfeeders | stop-logsearch | restart-solr |' \ - ' restart-logsearch | restart-ranger | restart-atlas | transport-old-data )' + ' restart-logsearch | restart-ranger | restart-atlas | transport-old-data )') sys.exit(1) command_elapsed_time = time.time() - command_start_time time_to_print = time.strftime("%H:%M:%S", time.gmtime(command_elapsed_time)) - print 30 * "-" - print "Command elapsed time: {0}".format(time_to_print) - print 30 * "-" - print "Migration helper command {0}FINISHED{1}".format(colors.OKGREEN, colors.ENDC) + print(30 * "-") + print("Command elapsed time: {0}".format(time_to_print)) + print(30 * "-") + print("Migration helper command {0}FINISHED{1}".format(colors.OKGREEN, colors.ENDC)) diff --git a/ambari-infra-solr-client/src/main/python/solrDataManager.py b/ambari-infra-solr-client/src/main/python/solrDataManager.py index e02c491..c3a0a1c 100755 --- a/ambari-infra-solr-client/src/main/python/solrDataManager.py +++ b/ambari-infra-solr-client/src/main/python/solrDataManager.py @@ -31,7 +31,7 @@ import tarfile import time from datetime import datetime, timedelta from subprocess import call, Popen, PIPE -from urllib import quote, unquote +from urllib.parse import quote, unquote from zipfile import ZipFile, ZIP_DEFLATED VERSION = "1.0" @@ -98,45 +98,45 @@ def parse_arguments(): for r in ["mode", "solr_url", "collection"]: if options.__dict__[r] is None: - print "argument '{0}' is mandatory".format(r) + print("argument '{0}' is mandatory".format(r)) parser.print_help() sys.exit() if not options.skip_date_usage: if options.filter_field is None: - print "argument 'filter_field' is mandatory" + print("argument 'filter_field' is mandatory") parser.print_help() sys.exit() mode_values = ["archive", "delete", "save"] if options.mode not in mode_values: - print "mode must be one of {0}".format(" | ".join(mode_values)) + print("mode must be one of {0}".format(" | ".join(mode_values))) parser.print_help() sys.exit() if options.mode == "delete": for r in ["name", "hdfs_keytab", "hdfs_principal", "hdfs_user", "hdfs_path", "key_file_path", "bucket", "key_prefix", "local_path"]: if options.__dict__[r] is not None: - print "argument '{0}' may not be specified in delete mode".format(r) + print("argument '{0}' may not be specified in delete mode".format(r)) parser.print_help() sys.exit() if not options.skip_date_usage and options.__dict__["end"] is None and options.__dict__["days"] is None or \ options.__dict__["end"] is not None and options.__dict__["days"] is not None: - print "exactly one of 'end' or 'days' must be specfied" + print("exactly one of 'end' or 'days' must be specfied") parser.print_help() sys.exit() is_any_solr_kerberos_property = options.__dict__["solr_keytab"] is not None or options.__dict__["solr_principal"] is not None is_all_solr_kerberos_property = options.__dict__["solr_keytab"] is not None and options.__dict__["solr_principal"] is not None if is_any_solr_kerberos_property and not is_all_solr_kerberos_property: - print "either both 'solr-keytab' and 'solr-principal' must be specfied, or neither of them" + print("either both 'solr-keytab' and 'solr-principal' must be specfied, or neither of them") parser.print_help() sys.exit() compression_values = ["none", "tar.gz", "tar.bz2", "zip", "gz"] if options.compression not in compression_values: - print "compression must be one of {0}".format(" | ".join(compression_values)) + print("compression must be one of {0}".format(" | ".join(compression_values))) parser.print_help() sys.exit() @@ -145,14 +145,14 @@ def parse_arguments(): is_any_hdfs_kerberos_property = options.__dict__["hdfs_keytab"] is not None or options.__dict__["hdfs_principal"] is not None is_all_hdfs_kerberos_property = options.__dict__["hdfs_keytab"] is not None and options.__dict__["hdfs_principal"] is not None if is_any_hdfs_kerberos_property and not is_all_hdfs_kerberos_property: - print "either both 'hdfs_keytab' and 'hdfs_principal' must be specfied, or neither of them" + print("either both 'hdfs_keytab' and 'hdfs_principal' must be specfied, or neither of them") parser.print_help() sys.exit() is_any_hdfs_property = options.__dict__["hdfs_user"] is not None or options.__dict__["hdfs_path"] is not None is_all_hdfs_property = options.__dict__["hdfs_user"] is not None and options.__dict__["hdfs_path"] is not None if is_any_hdfs_property and not is_all_hdfs_property: - print "either both 'hdfs_user' and 'hdfs_path' must be specfied, or neither of them" + print("either both 'hdfs_user' and 'hdfs_path' must be specfied, or neither of them") parser.print_help() sys.exit() @@ -161,7 +161,7 @@ def parse_arguments(): is_all_s3_property = options.__dict__["key_file_path"] is not None and options.__dict__["bucket"] is not None and \ options.__dict__["key_prefix"] is not None if is_any_s3_property and not is_all_s3_property: - print "either all the S3 arguments ('key_file_path', 'bucket', 'key_prefix') must be specfied, or none of them" + print("either all the S3 arguments ('key_file_path', 'bucket', 'key_prefix') must be specfied, or none of them") parser.print_help() sys.exit() @@ -169,70 +169,70 @@ def parse_arguments(): count = (1 if is_any_solr_output_property else 0) + (1 if is_any_hdfs_property else 0) + \ (1 if is_any_s3_property else 0) + (1 if options.__dict__["local_path"] is not None else 0) if count != 1: - print "exactly one of the HDFS arguments ('hdfs_user', 'hdfs_path') or the S3 arguments ('key_file_path', 'bucket', 'key_prefix') or the solr arguments ('solr_output_collection') or the 'local_path' argument must be specified" + print("exactly one of the HDFS arguments ('hdfs_user', 'hdfs_path') or the S3 arguments ('key_file_path', 'bucket', 'key_prefix') or the solr arguments ('solr_output_collection') or the 'local_path' argument must be specified") parser.print_help() sys.exit() if options.__dict__["hdfs_keytab"] is not None and options.__dict__["hdfs_user"] is None: - print "HDFS kerberos keytab and principal may only be specified if the upload target is HDFS" + print("HDFS kerberos keytab and principal may only be specified if the upload target is HDFS") parser.print_help() sys.exit() - print("You are running Solr Data Manager {0} with arguments:".format(VERSION)) - print(" mode: " + options.mode) - print(" solr-url: " + options.solr_url) - print(" collection: " + options.collection) + print(("You are running Solr Data Manager {0} with arguments:".format(VERSION))) + print((" mode: " + options.mode)) + print((" solr-url: " + options.solr_url)) + print((" collection: " + options.collection)) if options.__dict__["filter_field"] is not None: - print(" filter-field: " + options.filter_field) + print((" filter-field: " + options.filter_field)) if options.mode in ["archive", "save"]: - print(" id-field: " + options.id_field) + print((" id-field: " + options.id_field)) if options.__dict__["exclude_fields"] is not None: - print(" exclude fields: " + options.exclude_fields) + print((" exclude fields: " + options.exclude_fields)) if options.__dict__["end"] is not None: - print(" end: " + options.end) + print((" end: " + options.end)) else: - print(" days: " + str(options.days)) - print(" date-format: " + options.date_format) + print((" days: " + str(options.days))) + print((" date-format: " + options.date_format)) if options.__dict__["additional_filter"] is not None: - print(" additional-filter: " + str(options.additional_filter)) + print((" additional-filter: " + str(options.additional_filter))) if options.__dict__["name"] is not None: - print(" name: " + str(options.name)) + print((" name: " + str(options.name))) if options.mode in ["archive", "save"]: - print(" read-block-size: " + str(options.read_block_size)) - print(" write-block-size: " + str(options.write_block_size)) - print(" ignore-unfinished-uploading: " + str(options.ignore_unfinished_uploading)) + print((" read-block-size: " + str(options.read_block_size))) + print((" write-block-size: " + str(options.write_block_size))) + print((" ignore-unfinished-uploading: " + str(options.ignore_unfinished_uploading))) if (options.__dict__["solr_keytab"] is not None): - print(" solr-keytab: " + options.solr_keytab) - print(" solr-principal: " + options.solr_principal) + print((" solr-keytab: " + options.solr_keytab)) + print((" solr-principal: " + options.solr_principal)) if options.mode in ["archive", "save"]: - print(" output: " + ("json" if options.json_file else "line-delimited-json")) - print(" compression: " + options.compression) + print((" output: " + ("json" if options.json_file else "line-delimited-json"))) + print((" compression: " + options.compression)) if options.__dict__["solr_output_collection"] is not None: - print(" solr output collection: " + options.solr_output_collection) + print((" solr output collection: " + options.solr_output_collection)) if options.__dict__["solr_output_url"] is not None: - print(" solr output url: " + options.solr_output_collection) + print((" solr output url: " + options.solr_output_collection)) if (options.__dict__["hdfs_keytab"] is not None): - print(" hdfs-keytab: " + options.hdfs_keytab) - print(" hdfs-principal: " + options.hdfs_principal) + print((" hdfs-keytab: " + options.hdfs_keytab)) + print((" hdfs-principal: " + options.hdfs_principal)) if (options.__dict__["hdfs_user"] is not None): - print(" hdfs-user: " + options.hdfs_user) - print(" hdfs-path: " + options.hdfs_path) + print((" hdfs-user: " + options.hdfs_user)) + print((" hdfs-path: " + options.hdfs_path)) if (options.__dict__["key_file_path"] is not None): - print(" key-file-path: " + options.key_file_path) - print(" bucket: " + options.bucket) - print(" key-prefix: " + options.key_prefix) + print((" key-file-path: " + options.key_file_path)) + print((" bucket: " + options.bucket)) + print((" key-prefix: " + options.key_prefix)) if (options.__dict__["local_path"] is not None): - print(" local-path: " + options.local_path) - print (" skip-date-usage: " + str(options.skip_date_usage)) - print(" verbose: " + str(options.verbose)) - print + print((" local-path: " + options.local_path)) + print((" skip-date-usage: " + str(options.skip_date_usage))) + print((" verbose: " + str(options.verbose))) + print() if options.__dict__["additional_filter"] is not None and options.__dict__["name"] is None: go = False while not go: sys.stdout.write("It is recommended to set --name in case of any additional filter is set.\n") sys.stdout.write("Are you sure that you want to proceed without a name (yes/no)? ") - choice = raw_input().lower() + choice = input().lower() if choice in ['yes', 'ye', 'y']: go = True elif choice in ['no', 'n']: @@ -311,13 +311,13 @@ def ensure_hdfs_path(hdfs_kinit_command, hdfs_user, hdfs_path): logger.debug("Ensuring that the HDFS path %s exists:\n%s", hdfs_path, hdfs_create_dir_command) result = call(hdfs_create_dir_command.split()) except Exception as e: - print + print() logger.warn("Could not execute hdfs ensure dir command:\n%s", hdfs_create_dir_command) logger.warn(str(e)) sys.exit() if result != 0: - print + print() logger.warn("Could not ensure HDFS dir command:\n%s", hdfs_create_dir_command) logger.warn(str(err)) sys.exit() @@ -341,12 +341,12 @@ def handle_unfinished_uploading(solr_kinit_command, hdfs_kinit_command, curl_pre with open(command_json_path) as command_file: command = json.load(command_file) - if "upload" in command.keys() and ignore_unfinished_uploading: + if "upload" in list(command.keys()) and ignore_unfinished_uploading: logger.info("Ignoring unfinished uploading left by previous run") os.remove(command_json_path) return - if "upload" in command.keys(): + if "upload" in list(command.keys()): logger.info("Previous run has left unfinished uploading") logger.info("You may try to run the program with '-g' or '--ignore-unfinished-uploading' to ignore it if it keeps on failing") @@ -365,7 +365,7 @@ def handle_unfinished_uploading(solr_kinit_command, hdfs_kinit_command, curl_pre logger.warn("Unknown upload type: %s", command["upload"]["type"]) sys.exit() - if "delete" in command.keys(): + if "delete" in list(command.keys()): delete_data(solr_kinit_command, curl_prefix, command["delete"]["command"], command["delete"]["collection"], command["delete"]["filter_field"], command["delete"]["id_field"], command["delete"]["prev_lot_end_value"], command["delete"]["prev_lot_end_id"], skip_date_usage) @@ -469,7 +469,7 @@ def create_block(tmp_file_path, solr_kinit_command, curl_prefix, solr_query_url_ sys.stdout.write("\r{0} records are written".format(records)) sys.stdout.flush() if verbose and records < write_block_size: - print + print() logger.debug("Collecting next lot of data") finish_file(tmp_file, json_file) @@ -672,7 +672,7 @@ def upload_file_hdfs(hdfs_kinit_command, upload_command, upload_file_path, hdfs_ logger.debug("Checking if file already exists on hdfs:\n%s", hdfs_file_exists_command) hdfs_file_exists = (0 == call(hdfs_file_exists_command.split())) except Exception as e: - print + print() logger.warn("Could not execute command to check if file already exists on HDFS:\n%s", hdfs_file_exists_command) logger.warn(str(e)) sys.exit() @@ -682,7 +682,7 @@ def upload_file_hdfs(hdfs_kinit_command, upload_command, upload_file_path, hdfs_ logger.debug("Uploading file to hdfs:\n%s", upload_command) result = call(upload_command.split()) except Exception as e: - print + print() logger.warn("Could not execute command to upload file to HDFS:\n%s", upload_command) logger.warn(str(e)) sys.exit() @@ -700,7 +700,7 @@ def upload_file_s3(upload_command, upload_file_path, bucket, key_prefix): logger.debug("Uploading file to s3:\n%s", upload_command) result = call(upload_command.split()) except Exception as e: - print + print() logger.warn("Could not execute command to upload file to S3:\n%s", upload_command) logger.warn(str(e)) sys.exit() @@ -725,7 +725,7 @@ def upload_file_local(upload_command, upload_file_path, local_path): call(upload_command.split()) logger.info("File %s was moved to local directory %s", os.path.basename(upload_file_path), local_path) except Exception as e: - print + print() logger.warn("Could not execute move command command:\n%s", upload_command) logger.warn(str(e)) sys.exit() @@ -758,14 +758,14 @@ def query_solr(solr_kinit_command, url, curl_command, action, data=None): logger.debug("%s data from solr:\n%s", action, ' '.join(cmd)) process = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) except Exception as e: - print + print() logger.warn("Could not execute curl command:\n%s", ' '.join(cmd)) logger.warn(str(e)) sys.exit() out, err = process.communicate() if process.returncode != 0: - print + print() logger.warn("Could not execute curl command:\n%s", ' '.join(cmd)) logger.warn(str(err)) sys.exit() @@ -773,7 +773,7 @@ def query_solr(solr_kinit_command, url, curl_command, action, data=None): true = True # needed to be able to eval 'true' in the returned json rsp = eval(str(out)) if rsp["responseHeader"]["status"] != 0: - print + print() logger.warn("Could not execute solr query:\n%s", unquote(url)) logger.warn(rsp["error"]["msg"]) sys.exit() @@ -785,13 +785,13 @@ def run_kinit(kinit_command, program): logger.debug("Running kinit for %s:\n%s", program, kinit_command) result = call(kinit_command.split()) except Exception as e: - print + print() logger.warn("Could not execute %s kinit command:\n%s", program, kinit_command) logger.warn(str(e)) sys.exit() if result != 0: - print + print() logger.warn("%s kinit command was not successful:\n%s", program, kinit_command) sys.exit() @@ -817,7 +817,7 @@ if __name__ == '__main__': else: logger.warn("Unknown mode: %s", options.mode) - print("--- %s seconds ---" % (time.time() - start_time)) + print(("--- %s seconds ---" % (time.time() - start_time))) except KeyboardInterrupt: - print + print() sys.exit(128 + signal.SIGINT) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@ambari.apache.org For additional commands, e-mail: commits-h...@ambari.apache.org