AMBARI-10226: [WinTP2] Merge HDPWIN PIG package scripts to common services (jluniya)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/626ef03d Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/626ef03d Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/626ef03d Branch: refs/heads/trunk Commit: 626ef03daf847ad4e480ce5eb27fca7f6f9fcb81 Parents: 194df08 Author: Jayush Luniya <jlun...@hortonworks.com> Authored: Fri Mar 27 08:58:57 2015 -0700 Committer: Jayush Luniya <jlun...@hortonworks.com> Committed: Fri Mar 27 08:58:57 2015 -0700 ---------------------------------------------------------------------- .../PIG/0.12.0.2.0/package/scripts/params.py | 60 +-------------- .../0.12.0.2.0/package/scripts/params_linux.py | 78 ++++++++++++++++++++ .../package/scripts/params_windows.py | 36 +++++++++ .../PIG/0.12.0.2.0/package/scripts/pig.py | 20 ++++- .../0.12.0.2.0/package/scripts/pig_client.py | 26 +++++-- .../0.12.0.2.0/package/scripts/service_check.py | 16 ++++ .../2.1/services/PIG/package/scripts/params.py | 37 ---------- .../2.1/services/PIG/package/scripts/pig.py | 49 ------------ .../services/PIG/package/scripts/pig_client.py | 41 ---------- .../PIG/package/scripts/service_check.py | 34 --------- 10 files changed, 172 insertions(+), 225 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/626ef03d/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params.py index 2be49fa..9e3148e 100644 --- a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params.py @@ -18,61 +18,9 @@ limitations under the License. Ambari Agent """ +from ambari_commons import OSCheck -from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions -from resource_management import * - -# server configurations -config = Script.get_config() -tmp_dir = Script.get_tmp_dir() - -stack_name = default("/hostLevelParams/stack_name", None) - -stack_version_unformatted = str(config['hostLevelParams']['stack_version']) -hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) - -# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade -version = default("/commandParams/version", None) - -#hadoop params -if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0: - hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin" - hadoop_home = '/usr/hdp/current/hadoop-client' - pig_bin_dir = '/usr/hdp/current/pig-client/bin' +if OSCheck.is_windows_family(): + from params_windows import * else: - hadoop_bin_dir = "/usr/bin" - hadoop_home = '/usr' - pig_bin_dir = "" - -hadoop_conf_dir = "/etc/hadoop/conf" -pig_conf_dir = "/etc/pig/conf" -hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] -hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] -hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] -smokeuser = config['configurations']['cluster-env']['smokeuser'] -smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] -user_group = config['configurations']['cluster-env']['user_group'] -security_enabled = config['configurations']['cluster-env']['security_enabled'] -smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] -kinit_path_local = functions.get_kinit_path() -pig_env_sh_template = config['configurations']['pig-env']['content'] - -# not supporting 32 bit jdk. -java64_home = config['hostLevelParams']['java_home'] - -pig_properties = config['configurations']['pig-properties']['content'] - -log4j_props = config['configurations']['pig-log4j']['content'] - -import functools -#create partial functions with common arguments for every HdfsDirectory call -#to create hdfs directory we need to call params.HdfsDirectory in code -HdfsDirectory = functools.partial( - HdfsDirectory, - conf_dir=hadoop_conf_dir, - hdfs_user=hdfs_user, - security_enabled = security_enabled, - keytab = hdfs_user_keytab, - kinit_path_local = kinit_path_local, - bin_dir = hadoop_bin_dir -) + from params_linux import * http://git-wip-us.apache.org/repos/asf/ambari/blob/626ef03d/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_linux.py new file mode 100644 index 0000000..2be49fa --- /dev/null +++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_linux.py @@ -0,0 +1,78 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions +from resource_management import * + +# server configurations +config = Script.get_config() +tmp_dir = Script.get_tmp_dir() + +stack_name = default("/hostLevelParams/stack_name", None) + +stack_version_unformatted = str(config['hostLevelParams']['stack_version']) +hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) + +# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade +version = default("/commandParams/version", None) + +#hadoop params +if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0: + hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin" + hadoop_home = '/usr/hdp/current/hadoop-client' + pig_bin_dir = '/usr/hdp/current/pig-client/bin' +else: + hadoop_bin_dir = "/usr/bin" + hadoop_home = '/usr' + pig_bin_dir = "" + +hadoop_conf_dir = "/etc/hadoop/conf" +pig_conf_dir = "/etc/pig/conf" +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] +hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] +hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] +smokeuser = config['configurations']['cluster-env']['smokeuser'] +smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] +user_group = config['configurations']['cluster-env']['user_group'] +security_enabled = config['configurations']['cluster-env']['security_enabled'] +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] +kinit_path_local = functions.get_kinit_path() +pig_env_sh_template = config['configurations']['pig-env']['content'] + +# not supporting 32 bit jdk. +java64_home = config['hostLevelParams']['java_home'] + +pig_properties = config['configurations']['pig-properties']['content'] + +log4j_props = config['configurations']['pig-log4j']['content'] + +import functools +#create partial functions with common arguments for every HdfsDirectory call +#to create hdfs directory we need to call params.HdfsDirectory in code +HdfsDirectory = functools.partial( + HdfsDirectory, + conf_dir=hadoop_conf_dir, + hdfs_user=hdfs_user, + security_enabled = security_enabled, + keytab = hdfs_user_keytab, + kinit_path_local = kinit_path_local, + bin_dir = hadoop_bin_dir +) http://git-wip-us.apache.org/repos/asf/ambari/blob/626ef03d/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_windows.py new file mode 100644 index 0000000..3e63083 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_windows.py @@ -0,0 +1,36 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +from resource_management import * + +# server configurations +config = Script.get_config() +hdp_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"],"..")) +pig_home = os.environ['PIG_HOME'] +pig_conf_dir = os.path.join(pig_home,'conf') +pig_properties = config['configurations']['pig-properties']['content'] + +if (('pig-log4j' in config['configurations']) and ('content' in config['configurations']['pig-log4j'])): + log4j_props = config['configurations']['pig-log4j']['content'] +else: + log4j_props = None + +pig_user = "hadoop" +hdfs_user = "hadoop" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/626ef03d/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py index 036b930..c5f095f 100644 --- a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py +++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py @@ -19,9 +19,11 @@ Ambari Agent """ import os - from resource_management import * +from ambari_commons import OSConst +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) def pig(): import params @@ -58,3 +60,19 @@ def pig(): group=params.user_group, owner=params.hdfs_user ) + +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def pig(): + import params + File(os.path.join(params.pig_conf_dir, "pig.properties"), + mode="f", + owner=params.pig_user, + content=params.pig_properties + ) + + if (params.log4j_props != None): + File(os.path.join(params.pig_conf_dir, "log4j.properties"), + mode='f', + owner=params.pig_user, + content=params.log4j_props + ) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/626ef03d/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig_client.py b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig_client.py index eec9304..5b81b1f 100644 --- a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig_client.py +++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig_client.py @@ -20,12 +20,23 @@ Ambari Agent """ import sys +import os from resource_management import * from pig import pig - +from ambari_commons import OSConst +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl class PigClient(Script): + def configure(self, env): + import params + env.set_params(params) + pig() + def status(self, env): + raise ClientComponentHasNoStatus() + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class PigClientLinux(PigClient): def get_stack_to_component(self): return {"HDP": "hadoop-client"} @@ -40,13 +51,14 @@ class PigClient(Script): self.install_packages(env) self.configure(env) - def configure(self, env): - import params - env.set_params(params) - pig() +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class PigClientWindows(PigClient): - def status(self, env): - raise ClientComponentHasNoStatus() + def install(self, env): + import params + if params.pig_home is None: + self.install_packages(env) + self.configure(env) if __name__ == "__main__": PigClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/626ef03d/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/service_check.py index 4fbfeb4..9de30ed 100644 --- a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/service_check.py +++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/service_check.py @@ -21,8 +21,15 @@ Ambari Agent from resource_management import * from resource_management.libraries.functions.dynamic_variable_interpretation import copy_tarballs_to_hdfs +from resource_management.libraries import functions +from ambari_commons import OSConst +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl class PigServiceCheck(Script): + pass + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class PigServiceCheckLinux(PigServiceCheck): def service_check(self, env): import params env.set_params(params) @@ -104,6 +111,15 @@ class PigServiceCheck(Script): bin_dir = params.hadoop_bin_dir ) +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class PigServiceCheckWindows(PigServiceCheck): + def service_check(self, env): + import params + env.set_params(params) + smoke_cmd = os.path.join(params.hdp_root,"Run-SmokeTests.cmd") + service = "PIG" + Execute(format("cmd /C {smoke_cmd} {service}"), logoutput=True, user=params.hdfs_user) + if __name__ == "__main__": PigServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/626ef03d/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/params.py deleted file mode 100644 index 854d577..0000000 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/params.py +++ /dev/null @@ -1,37 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -from resource_management import * - -# server configurations -config = Script.get_config() -hdp_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"],"..")) -pig_home = os.environ['PIG_HOME'] -pig_conf_dir = os.path.join(pig_home,'conf') -pig_properties = config['configurations']['pig-properties']['pig-content'] - -if (('pig-log4j' in config['configurations']) and ('content' in config['configurations']['pig-log4j'])): - log4j_props = config['configurations']['pig-log4j']['content'] -else: - log4j_props = None - -pig_user = "hadoop" -hdfs_user = "hadoop" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/626ef03d/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/pig.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/pig.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/pig.py deleted file mode 100644 index 47bb40f..0000000 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/pig.py +++ /dev/null @@ -1,49 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -import os -from resource_management import * - -def pig(): - import params - File(os.path.join(params.pig_conf_dir, "pig.properties"), - mode="f", - owner=params.pig_user, - content=params.pig_properties - ) - - if (params.log4j_props != None): - File(os.path.join(params.pig_conf_dir, "log4j.properties"), - mode='f', - owner=params.pig_user, - content=params.log4j_props - ) - -def pig_TemplateConfig(name): - import params - - if not isinstance(name, list): - name = [name] - - for x in name: - TemplateConfig(os.path.join(params.pig_conf_dir,x), - owner = params.hdfs_user - ) http://git-wip-us.apache.org/repos/asf/ambari/blob/626ef03d/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/pig_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/pig_client.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/pig_client.py deleted file mode 100644 index 381bfe9..0000000 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/pig_client.py +++ /dev/null @@ -1,41 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -import os -from resource_management import * - - -class PigClient(Script): - def install(self, env): - import params - if params.pig_home is None: - self.install_packages(env) - self.configure(env) - - def configure(self, env): - import params - env.set_params(params) - - def status(self, env): - raise ClientComponentHasNoStatus() - -if __name__ == "__main__": - PigClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/626ef03d/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/service_check.py deleted file mode 100644 index 9e519c5..0000000 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/service_check.py +++ /dev/null @@ -1,34 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -from resource_management import * -from resource_management.libraries import functions - -class PigServiceCheck(Script): - def service_check(self, env): - import params - env.set_params(params) - smoke_cmd = os.path.join(params.hdp_root,"Run-SmokeTests.cmd") - service = "PIG" - Execute(format("cmd /C {smoke_cmd} {service}"), logoutput=True, user=params.hdfs_user) - -if __name__ == "__main__": - PigServiceCheck().execute()