Repository: ambari Updated Branches: refs/heads/branch-2.1 9a7d9b94a -> b6c07db16
AMBARI-12954. upgradeHelper need to support Ambari 1.6.1 (backup functionality).(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b6c07db1 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b6c07db1 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b6c07db1 Branch: refs/heads/branch-2.1 Commit: b6c07db1654c96336ea820f3ece795b968e20538 Parents: 9a7d9b9 Author: Vitaly Brodetskyi <vbrodets...@hortonworks.com> Authored: Tue Sep 1 12:44:12 2015 +0300 Committer: Vitaly Brodetskyi <vbrodets...@hortonworks.com> Committed: Tue Sep 1 12:44:12 2015 +0300 ---------------------------------------------------------------------- ambari-server/src/main/python/upgradeHelper.py | 79 ++++++++++++++++----- 1 file changed, 62 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b6c07db1/ambari-server/src/main/python/upgradeHelper.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/upgradeHelper.py b/ambari-server/src/main/python/upgradeHelper.py index 5a24b6f..4189b95 100644 --- a/ambari-server/src/main/python/upgradeHelper.py +++ b/ambari-server/src/main/python/upgradeHelper.py @@ -225,6 +225,8 @@ class Options(Const): """:type : ServerConfigFactory""" stack_advisor = None """:type : StackAdvisor""" + ambari_server = None + """:type : AmbariServer""" # Api constants ROOT_URL = None @@ -234,6 +236,7 @@ class Options(Const): # Curl options CURL_PRINT_ONLY = None + CURL_WRITE_ONLY = None ARGS = None OPTIONS = None @@ -269,6 +272,16 @@ class Options(Const): cls.SERVICES = set(map(lambda x: x.upper(), get_cluster_services())) cls.ambari_server = AmbariServer() + if not cls.isPropertyAttributesSupported(): + cls.logger.warning("Property attributes not supported by current Ambari version") + + @classmethod + def isPropertyAttributesSupported(cls): + if cls.ambari_server.server_version[0] * 10 + cls.ambari_server.server_version[1] >= 17: + return True + return False + + @classmethod def initialize_logger(cls, filename=None): @@ -1203,7 +1216,7 @@ def delete_mr(): def get_cluster_stackname(): VERSION_URL_FORMAT = Options.CLUSTER_URL + '?fields=Clusters/version' - structured_resp = curl(VERSION_URL_FORMAT, simulate=False, validate=True, parse=True) + structured_resp = curl(VERSION_URL_FORMAT, validate=True, parse=True) if 'Clusters' in structured_resp: if 'version' in structured_resp['Clusters']: @@ -1218,7 +1231,7 @@ def has_component_in_stack_def(stack_name, service_name, component_name): try: curl(STACK_COMPONENT_URL_FORMAT.format(stack, stack_version, service_name, component_name), - validate=True, simulate=False) + validate=True) return True except FatalException: return False @@ -1271,11 +1284,24 @@ def update_config(properties, config_type, attributes=None): curl(Options.CLUSTER_URL, request_type="PUT", data=properties_payload, validate=True, soft_validation=True) +def build_all_options(desired_configs): + """ + Get all configs in the old-fashion way ( versions below 1.7.0 doesn't support "properties" filter ) + """ + config_url_tpl = Options.CLUSTER_URL + "/configurations?type={0}&tag={1}" + all_options = {CatConst.ITEMS_TAG: []} + for config in desired_configs: + cfg_item = curl(config_url_tpl.format(config, desired_configs[config]["tag"]), parse=True, validate=True) + if CatConst.ITEMS_TAG in cfg_item and len(cfg_item[CatConst.ITEMS_TAG]) == 1: + all_options[CatConst.ITEMS_TAG].append(cfg_item[CatConst.ITEMS_TAG][0]) + + return all_options + + def get_config_resp_all(): desired_configs = {} - config_all_properties_url = Options.CLUSTER_URL + "/configurations?fields=properties,properties_attributes" - desired_configs_resp = curl(Options.CLUSTER_URL + "?fields=Clusters/desired_configs", validate=True, parse=True, simulate=False) - all_options = curl(config_all_properties_url, validate=True, parse=True, simulate=False) + config_all_properties_url = Options.CLUSTER_URL + "/configurations?fields=properties" + desired_configs_resp = curl(Options.CLUSTER_URL + "?fields=Clusters/desired_configs", validate=True, parse=True) if 'Clusters' in desired_configs_resp: if 'desired_configs' in desired_configs_resp['Clusters']: @@ -1285,6 +1311,12 @@ def get_config_resp_all(): else: return None + if Options.isPropertyAttributesSupported(): + config_all_properties_url += ",properties_attributes" + all_options = curl(config_all_properties_url, validate=True, parse=True) + else: + all_options = build_all_options(desired_configs_resp) + if CatConst.ITEMS_TAG in all_options: all_options = all_options[CatConst.ITEMS_TAG] else: @@ -1330,7 +1362,7 @@ def is_services_exists(required_services): def get_cluster_services(): services_url = Options.CLUSTER_URL + '/services' - raw_services = curl(services_url, parse=True, simulate=False) + raw_services = curl(services_url, parse=True) # expected structure: # items: [ {"href":"...", "ServiceInfo":{"cluster_name":"..", "service_name":".."}}, ..., ... ] @@ -1342,7 +1374,7 @@ def get_cluster_services(): def get_zookeeper_quorum(): - zoo_cfg = curl(Options.COMPONENTS_FORMAT.format(Options.ZOOKEEPER_SERVER), validate=False, simulate=False, parse=True) + zoo_cfg = curl(Options.COMPONENTS_FORMAT.format(Options.ZOOKEEPER_SERVER), validate=False, parse=True) zoo_quorum = [] zoo_def_port = "2181" if Options.server_config_factory is not None and Options.ZK_OPTIONS in Options.server_config_factory.items(): @@ -1359,7 +1391,7 @@ def get_zookeeper_quorum(): def get_tez_history_url_base(): try: - tez_view = curl(Options.TEZ_VIEW_URL, validate=False, simulate=False, parse=True) + tez_view = curl(Options.TEZ_VIEW_URL, validate=False, parse=True) except HTTPError as e: raise TemplateProcessingException(str(e)) @@ -1389,7 +1421,7 @@ def get_kafka_listeners(): def get_ranger_xaaudit_hdfs_destination_directory(): namenode_hostname="localhost" - namenode_cfg = curl(Options.COMPONENTS_FORMAT.format(Options.NAMENODE), validate=False, simulate=False, parse=True) + namenode_cfg = curl(Options.COMPONENTS_FORMAT.format(Options.NAMENODE), validate=False, parse=True) if "host_components" in namenode_cfg: namenode_hostname = namenode_cfg["host_components"][0]["HostRoles"]["host_name"] @@ -1491,7 +1523,7 @@ def get_jh_host(catalog): return "" def get_ranger_host(): - ranger_config = curl(Options.COMPONENTS_FORMAT.format('RANGER_ADMIN'), validate=False, simulate=False, parse=True) + ranger_config = curl(Options.COMPONENTS_FORMAT.format('RANGER_ADMIN'), validate=False, parse=True) ranger_host_list = [] if "host_components" in ranger_config: for item in ranger_config["host_components"]: @@ -1856,14 +1888,18 @@ def generate_auth_header(user, password): def curl(url, tokens=None, headers=None, request_type="GET", data=None, parse=False, - simulate=None, validate=False, soft_validation=False): + validate=False, soft_validation=False): + """ + :rtype type + """ _headers = {} handler_chain = [] post_req = ["POST", "PUT"] get_req = ["GET", "DELETE"] - simulate_only = Options.CURL_PRINT_ONLY is not None or (simulate is not None and simulate is True) - print_url = Options.CURL_PRINT_ONLY is not None and simulate is not None + print_url = Options.CURL_PRINT_ONLY is not None + write_only_print = Options.CURL_WRITE_ONLY is not None + if request_type not in post_req + get_req: raise IOError("Wrong request type \"%s\" passed" % request_type) @@ -1894,10 +1930,14 @@ def curl(url, tokens=None, headers=None, request_type="GET", data=None, parse=Fa req.get_method = lambda: request_type if print_url: - Options.logger.info(url) + if write_only_print: + if request_type in post_req: + Options.logger.info(url) + else: + Options.logger.info(url) code = 200 - if not simulate_only: + if not (print_url and request_type in post_req): try: resp = director.open(req) out = resp.read() @@ -1915,7 +1955,7 @@ def curl(url, tokens=None, headers=None, request_type="GET", data=None, parse=Fa Options.logger.info(url) out = "{}" - if validate and not simulate_only and (code > 299 or code < 200): + if validate and not print_url and (code > 299 or code < 200): if soft_validation: Options.logger.warning("Response validation failed, please check previous action result manually.") else: @@ -2112,7 +2152,10 @@ def main(): parser.add_option("-n", "--printonly", action="store_true", dest="printonly", default=False, - help="Prints all the curl commands to be executed (only for write/update actions)") + help="Prints all the curl commands to be executed (no post/update request will be performed)") + parser.add_option("-w", "--writeonly", + action="store_true", dest="writeonly", default=False, + help="in the combination with --printonly param will print only post/update requests") parser.add_option("-o", "--log", dest="logfile", default=None, help="Log file") parser.add_option("--report", dest="report", default=None, @@ -2178,6 +2221,8 @@ def main(): if options.printonly: Options.CURL_PRINT_ONLY = "yes" options.exit_message = "Simulated execution of action '%s'. Verify the list edit calls." % action + if options.writeonly: + Options.CURL_WRITE_ONLY = "yes" Options.ARGS = args Options.OPTIONS = options