http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/balancer-emulator/balancer.log ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/balancer-emulator/balancer.log b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/balancer-emulator/balancer.log new file mode 100644 index 0000000..2010c02 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/balancer-emulator/balancer.log @@ -0,0 +1,29 @@ +Time Stamp Iteration# Bytes Already Moved Bytes Left To Move Bytes Being Moved +Jul 28, 2014 5:01:49 PM 0 0 B 5.74 GB 9.79 GB +Jul 28, 2014 5:03:00 PM 1 0 B 5.58 GB 9.79 GB +Jul 28, 2014 5:04:07 PM 2 0 B 5.40 GB 9.79 GB +Jul 28, 2014 5:05:14 PM 3 0 B 5.06 GB 9.79 GB +Jul 28, 2014 5:05:50 PM 4 0 B 5.06 GB 9.79 GB +Jul 28, 2014 5:06:56 PM 5 0 B 4.81 GB 9.79 GB +Jul 28, 2014 5:07:33 PM 6 0 B 4.80 GB 9.79 GB +Jul 28, 2014 5:09:11 PM 7 0 B 4.29 GB 9.79 GB +Jul 28, 2014 5:09:47 PM 8 0 B 4.29 GB 9.79 GB +Jul 28, 2014 5:11:24 PM 9 0 B 3.89 GB 9.79 GB +Jul 28, 2014 5:12:00 PM 10 0 B 3.86 GB 9.79 GB +Jul 28, 2014 5:13:37 PM 11 0 B 3.23 GB 9.79 GB +Jul 28, 2014 5:15:13 PM 12 0 B 2.53 GB 9.79 GB +Jul 28, 2014 5:15:49 PM 13 0 B 2.52 GB 9.79 GB +Jul 28, 2014 5:16:25 PM 14 0 B 2.51 GB 9.79 GB +Jul 28, 2014 5:17:01 PM 15 0 B 2.39 GB 9.79 GB +Jul 28, 2014 5:17:37 PM 16 0 B 2.38 GB 9.79 GB +Jul 28, 2014 5:18:14 PM 17 0 B 2.31 GB 9.79 GB +Jul 28, 2014 5:18:50 PM 18 0 B 2.30 GB 9.79 GB +Jul 28, 2014 5:19:26 PM 19 0 B 2.21 GB 9.79 GB +Jul 28, 2014 5:20:02 PM 20 0 B 2.10 GB 9.79 GB +Jul 28, 2014 5:20:38 PM 21 0 B 2.06 GB 9.79 GB +Jul 28, 2014 5:22:14 PM 22 0 B 1.68 GB 9.79 GB +Jul 28, 2014 5:23:20 PM 23 0 B 1.00 GB 9.79 GB +Jul 28, 2014 5:23:56 PM 24 0 B 1016.16 MB 9.79 GB +Jul 28, 2014 5:25:33 PM 25 0 B 30.55 MB 9.79 GB +The cluster is balanced. Exiting... +Balancing took 24.858033333333335 minutes
http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/balancer-emulator/hdfs-command.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/balancer-emulator/hdfs-command.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/balancer-emulator/hdfs-command.py new file mode 100644 index 0000000..0cce48c --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/balancer-emulator/hdfs-command.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' +import time +import sys +from threading import Thread + + +def write_function(path, handle, interval): + with open(path) as f: + for line in f: + handle.write(line) + handle.flush() + time.sleep(interval) + +thread = Thread(target = write_function, args = ('balancer.log', sys.stdout, 1.5)) +thread.start() + +threaderr = Thread(target = write_function, args = ('balancer-err.log', sys.stderr, 1.5 * 0.023)) +threaderr.start() + +thread.join() + + +def rebalancer_out(): + write_function('balancer.log', sys.stdout) + +def rebalancer_err(): + write_function('balancer-err.log', sys.stdout) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/datanode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/datanode.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/datanode.py new file mode 100644 index 0000000..c1a66fb --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/datanode.py @@ -0,0 +1,59 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +from hdfs_datanode import datanode +from hdfs import hdfs + + +class DataNode(Script): + def install(self, env): + import params + + self.install_packages(env, params.exclude_packages) + env.set_params(params) + + def start(self, env): + import params + + env.set_params(params) + self.configure(env) + datanode(action="start") + + def stop(self, env): + import params + + env.set_params(params) + datanode(action="stop") + + def configure(self, env): + import params + env.set_params(params) + hdfs() + datanode(action="configure") + + def status(self, env): + import status_params + + env.set_params(status_params) + check_process_status(status_params.datanode_pid_file) + + +if __name__ == "__main__": + DataNode().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs.py new file mode 100644 index 0000000..873aa15 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs.py @@ -0,0 +1,80 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management import * +import sys +import os + + +def hdfs(name=None): + import params + + # On some OS this folder could be not exists, so we will create it before pushing there files + Directory(params.limits_conf_dir, + recursive=True, + owner='root', + group='root' + ) + + File(os.path.join(params.limits_conf_dir, 'hdfs.conf'), + owner='root', + group='root', + mode=0644, + content=Template("hdfs.conf.j2") + ) + + if params.security_enabled: + tc_mode = 0644 + tc_owner = "root" + else: + tc_mode = None + tc_owner = params.hdfs_user + + if "hadoop-policy" in params.config['configurations']: + XmlConfig("hadoop-policy.xml", + conf_dir=params.hadoop_conf_dir, + configurations=params.config['configurations']['hadoop-policy'], + configuration_attributes=params.config['configuration_attributes']['hadoop-policy'], + owner=params.hdfs_user, + group=params.user_group + ) + + XmlConfig("hdfs-site.xml", + conf_dir=params.hadoop_conf_dir, + configurations=params.config['configurations']['hdfs-site'], + configuration_attributes=params.config['configuration_attributes']['hdfs-site'], + owner=params.hdfs_user, + group=params.user_group + ) + + XmlConfig("core-site.xml", + conf_dir=params.hadoop_conf_dir, + configurations=params.config['configurations']['core-site'], + configuration_attributes=params.config['configuration_attributes']['core-site'], + owner=params.hdfs_user, + group=params.user_group, + mode=0644 + ) + + File(os.path.join(params.hadoop_conf_dir, 'slaves'), + owner=tc_owner, + content=Template("slaves.j2") + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_client.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_client.py new file mode 100644 index 0000000..3b4cf3e --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_client.py @@ -0,0 +1,53 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +from hdfs import hdfs +from utils import service + + +class HdfsClient(Script): + def install(self, env): + import params + + self.install_packages(env, params.exclude_packages) + env.set_params(params) + self.config(env) + + def start(self, env): + import params + + env.set_params(params) + + def stop(self, env): + import params + + env.set_params(params) + + def status(self, env): + raise ClientComponentHasNoStatus() + + def config(self, env): + import params + hdfs() + pass + + +if __name__ == "__main__": + HdfsClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_datanode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_datanode.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_datanode.py new file mode 100644 index 0000000..c93c6e4 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_datanode.py @@ -0,0 +1,56 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +from resource_management.libraries.functions.dfs_datanode_helper import handle_dfs_data_dir +from utils import service + + +def create_dirs(data_dir, params): + """ + :param data_dir: The directory to create + :param params: parameters + """ + Directory(data_dir, + recursive=True, + mode=0755, + owner=params.hdfs_user, + group=params.user_group, + ignore_failures=True + ) + + +def datanode(action=None): + import params + if action == "configure": + Directory(params.dfs_domain_socket_dir, + recursive=True, + mode=0751, + owner=params.hdfs_user, + group=params.user_group) + + handle_dfs_data_dir(create_dirs, params) + + elif action == "start" or action == "stop": + service( + action=action, name="datanode", + user=params.hdfs_user, + create_pid_dir=True, + create_log_dir=True + ) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_namenode.py new file mode 100644 index 0000000..31fc2f1 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_namenode.py @@ -0,0 +1,160 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +from utils import service + + +def namenode(action=None, do_format=True): + import params + #we need this directory to be present before any action(HA manual steps for + #additional namenode) + if action == "configure": + create_name_dirs(params.dfs_name_dir) + + if action == "start": + if do_format: + format_namenode() + pass + + File(params.exclude_file_path, + content=Template("exclude_hosts_list.j2"), + owner=params.hdfs_user, + group=params.user_group + ) + + service( + action="start", name="namenode", user=params.hdfs_user, + create_pid_dir=True, + create_log_dir=True + ) + if params.dfs_ha_enabled: + dfs_check_nn_status_cmd = format("su -s /bin/bash - {hdfs_user} -c 'export PATH=$PATH:{hadoop_bin_dir} ; hdfs --config {hadoop_conf_dir} haadmin -getServiceState {namenode_id} | grep active > /dev/null'") + else: + dfs_check_nn_status_cmd = None + + namenode_safe_mode_off = format("su -s /bin/bash - {hdfs_user} -c 'export PATH=$PATH:{hadoop_bin_dir} ; hadoop --config {hadoop_conf_dir} dfsadmin -safemode get' | grep 'Safe mode is OFF'") + + if params.security_enabled: + Execute(format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name}"), + user = params.hdfs_user) + Execute(namenode_safe_mode_off, + tries=40, + try_sleep=10, + only_if=dfs_check_nn_status_cmd #skip when HA not active + ) + create_hdfs_directories(dfs_check_nn_status_cmd) + if action == "stop": + service( + action="stop", name="namenode", + user=params.hdfs_user + ) + + if action == "decommission": + decommission() + +def create_name_dirs(directories): + import params + + dirs = directories.split(",") + Directory(dirs, + mode=0755, + owner=params.hdfs_user, + group=params.user_group, + recursive=True + ) + + +def create_hdfs_directories(check): + import params + + params.HdfsDirectory("/tmp", + action="create_delayed", + owner=params.hdfs_user, + mode=0777 + ) + params.HdfsDirectory(params.smoke_hdfs_user_dir, + action="create_delayed", + owner=params.smoke_user, + mode=params.smoke_hdfs_user_mode + ) + params.HdfsDirectory(None, action="create", + only_if=check #skip creation when HA not active + ) + +def format_namenode(force=None): + import params + + old_mark_dir = params.namenode_formatted_old_mark_dir + mark_dir = params.namenode_formatted_mark_dir + dfs_name_dir = params.dfs_name_dir + hdfs_user = params.hdfs_user + hadoop_conf_dir = params.hadoop_conf_dir + + if not params.dfs_ha_enabled: + if force: + ExecuteHadoop('namenode -format', + kinit_override=True, + bin_dir=params.hadoop_bin_dir, + conf_dir=hadoop_conf_dir) + else: + File(format("{tmp_dir}/checkForFormat.sh"), + content=StaticFile("checkForFormat.sh"), + mode=0755) + Execute(format( + "{tmp_dir}/checkForFormat.sh {hdfs_user} {hadoop_conf_dir} " + "{hadoop_bin_dir} {old_mark_dir} {mark_dir} {dfs_name_dir}"), + not_if=format("test -d {old_mark_dir} || test -d {mark_dir}"), + path="/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin" + ) + + Directory(mark_dir, + recursive = True + ) + + +def decommission(): + import params + + hdfs_user = params.hdfs_user + conf_dir = params.hadoop_conf_dir + user_group = params.user_group + dn_kinit_cmd = params.dn_kinit_cmd + + File(params.exclude_file_path, + content=Template("exclude_hosts_list.j2"), + owner=hdfs_user, + group=user_group + ) + + Execute(dn_kinit_cmd, + user=hdfs_user + ) + + if params.dfs_ha_enabled: + # due to a bug in hdfs, refreshNodes will not run on both namenodes so we + # need to execute each command scoped to a particular namenode + nn_refresh_cmd = format('dfsadmin -fs hdfs://{namenode_rpc} -refreshNodes') + else: + nn_refresh_cmd = format('dfsadmin -refreshNodes') + ExecuteHadoop(nn_refresh_cmd, + user=hdfs_user, + conf_dir=conf_dir, + kinit_override=True, + bin_dir=params.hadoop_bin_dir) http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_rebalance.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_rebalance.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_rebalance.py new file mode 100644 index 0000000..1dc545e --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_rebalance.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import re + +class HdfsParser(): + def __init__(self): + self.initialLine = None + self.state = None + + def parseLine(self, line): + hdfsLine = HdfsLine() + type, matcher = hdfsLine.recognizeType(line) + if(type == HdfsLine.LineType.HeaderStart): + self.state = 'PROCESS_STARTED' + elif (type == HdfsLine.LineType.Progress): + self.state = 'PROGRESS' + hdfsLine.parseProgressLog(line, matcher) + if(self.initialLine == None): self.initialLine = hdfsLine + + return hdfsLine + elif (type == HdfsLine.LineType.ProgressEnd): + self.state = 'PROCESS_FINISED' + return None + +class HdfsLine(): + + class LineType: + HeaderStart, Progress, ProgressEnd, Unknown = range(4) + + + MEMORY_SUFFIX = ['B','KB','MB','GB','TB','PB','EB'] + MEMORY_PATTERN = '(?P<memmult_%d>(?P<memory_%d>(\d+)(.|,)?(\d+)?) (?P<mult_%d>'+"|".join(MEMORY_SUFFIX)+'))' + + HEADER_BEGIN_PATTERN = re.compile('Time Stamp\w+Iteration#\w+Bytes Already Moved\w+Bytes Left To Move\w+Bytes Being Moved') + PROGRESS_PATTERN = re.compile( + "(?P<date>.*?)\s+" + + "(?P<iteration>\d+)\s+" + + MEMORY_PATTERN % (1,1,1) + "\s+" + + MEMORY_PATTERN % (2,2,2) + "\s+" + + MEMORY_PATTERN % (3,3,3) + ) + PROGRESS_END_PATTERN = re.compile('(The cluster is balanced. Exiting...|The cluster is balanced. Exiting...)') + + def __init__(self): + self.date = None + self.iteration = None + self.bytesAlreadyMoved = None + self.bytesLeftToMove = None + self.bytesBeingMoved = None + self.bytesAlreadyMovedStr = None + self.bytesLeftToMoveStr = None + self.bytesBeingMovedStr = None + + def recognizeType(self, line): + for (type, pattern) in ( + (HdfsLine.LineType.HeaderStart, self.HEADER_BEGIN_PATTERN), + (HdfsLine.LineType.Progress, self.PROGRESS_PATTERN), + (HdfsLine.LineType.ProgressEnd, self.PROGRESS_END_PATTERN) + ): + m = re.match(pattern, line) + if m: + return type, m + return HdfsLine.LineType.Unknown, None + + def parseProgressLog(self, line, m): + ''' + Parse the line of 'hdfs rebalancer' output. The example output being parsed: + + Time Stamp Iteration# Bytes Already Moved Bytes Left To Move Bytes Being Moved + Jul 28, 2014 5:01:49 PM 0 0 B 5.74 GB 9.79 GB + Jul 28, 2014 5:03:00 PM 1 0 B 5.58 GB 9.79 GB + + Throws AmbariException in case of parsing errors + + ''' + m = re.match(self.PROGRESS_PATTERN, line) + if m: + self.date = m.group('date') + self.iteration = int(m.group('iteration')) + + self.bytesAlreadyMoved = self.parseMemory(m.group('memory_1'), m.group('mult_1')) + self.bytesLeftToMove = self.parseMemory(m.group('memory_2'), m.group('mult_2')) + self.bytesBeingMoved = self.parseMemory(m.group('memory_3'), m.group('mult_3')) + + self.bytesAlreadyMovedStr = m.group('memmult_1') + self.bytesLeftToMoveStr = m.group('memmult_2') + self.bytesBeingMovedStr = m.group('memmult_3') + else: + raise AmbariException("Failed to parse line [%s]") + + def parseMemory(self, memorySize, multiplier_type): + try: + factor = self.MEMORY_SUFFIX.index(multiplier_type) + except ValueError: + raise AmbariException("Failed to memory value [%s %s]" % (memorySize, multiplier_type)) + + return float(memorySize) * (1024 ** factor) + def toJson(self): + return { + 'timeStamp' : self.date, + 'iteration' : self.iteration, + + 'dataMoved': self.bytesAlreadyMovedStr, + 'dataLeft' : self.bytesLeftToMoveStr, + 'dataBeingMoved': self.bytesBeingMovedStr, + + 'bytesMoved': self.bytesAlreadyMoved, + 'bytesLeft' : self.bytesLeftToMove, + 'bytesBeingMoved': self.bytesBeingMoved, + } + def __str__(self): + return "[ date=%s,iteration=%d, bytesAlreadyMoved=%d, bytesLeftToMove=%d, bytesBeingMoved=%d]"%(self.date, self.iteration, self.bytesAlreadyMoved, self.bytesLeftToMove, self.bytesBeingMoved) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_snamenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_snamenode.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_snamenode.py new file mode 100644 index 0000000..49241b4 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_snamenode.py @@ -0,0 +1,45 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +from utils import service +from utils import hdfs_directory + + +def snamenode(action=None, format=False): + import params + + if action == "configure": + Directory(params.fs_checkpoint_dir, + recursive=True, + mode=0755, + owner=params.hdfs_user, + group=params.user_group) + File(params.exclude_file_path, + content=Template("exclude_hosts_list.j2"), + owner=params.hdfs_user, + group=params.user_group) + elif action == "start" or action == "stop": + service( + action=action, + name="secondarynamenode", + user=params.hdfs_user, + create_pid_dir=True, + create_log_dir=True + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/journalnode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/journalnode.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/journalnode.py new file mode 100644 index 0000000..de18c88 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/journalnode.py @@ -0,0 +1,73 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +from utils import service +from hdfs import hdfs + + +class JournalNode(Script): + def install(self, env): + import params + + self.install_packages(env, params.exclude_packages) + env.set_params(params) + + def start(self, env): + import params + + env.set_params(params) + self.configure(env) + service( + action="start", name="journalnode", user=params.hdfs_user, + create_pid_dir=True, + create_log_dir=True + ) + + def stop(self, env): + import params + + env.set_params(params) + service( + action="stop", name="journalnode", user=params.hdfs_user, + create_pid_dir=True, + create_log_dir=True + ) + + def configure(self, env): + import params + + Directory(params.jn_edits_dir, + recursive=True, + owner=params.hdfs_user, + group=params.user_group + ) + env.set_params(params) + hdfs() + pass + + def status(self, env): + import status_params + + env.set_params(status_params) + check_process_status(status_params.journalnode_pid_file) + + +if __name__ == "__main__": + JournalNode().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/namenode.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/namenode.py new file mode 100644 index 0000000..a0b07aa --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/namenode.py @@ -0,0 +1,134 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +from hdfs_namenode import namenode +from hdfs import hdfs +import time +import json +import subprocess +import hdfs_rebalance +import sys +import os +from datetime import datetime + + +class NameNode(Script): + def install(self, env): + import params + + self.install_packages(env, params.exclude_packages) + env.set_params(params) + #TODO we need this for HA because of manual steps + self.configure(env) + + def start(self, env): + import params + + env.set_params(params) + self.configure(env) + namenode(action="start") + + def stop(self, env): + import params + + env.set_params(params) + namenode(action="stop") + + def configure(self, env): + import params + + env.set_params(params) + hdfs() + namenode(action="configure") + pass + + def status(self, env): + import status_params + + env.set_params(status_params) + Execute(format("echo '{namenode_pid_file}' >> /1.txt")) + check_process_status(status_params.namenode_pid_file) + pass + + def decommission(self, env): + import params + + env.set_params(params) + namenode(action="decommission") + pass + + + def rebalancehdfs(self, env): + import params + env.set_params(params) + + name_node_parameters = json.loads( params.name_node_params ) + threshold = name_node_parameters['threshold'] + _print("Starting balancer with threshold = %s\n" % threshold) + + def calculateCompletePercent(first, current): + return 1.0 - current.bytesLeftToMove/first.bytesLeftToMove + + + def startRebalancingProcess(threshold): + rebalanceCommand = format('export PATH=$PATH:{hadoop_bin_dir} ; hadoop --config {hadoop_conf_dir} balancer -threshold {threshold}') + return ['su','-',params.hdfs_user,'-c', rebalanceCommand] + + command = startRebalancingProcess(threshold) + + basedir = os.path.join(env.config.basedir, 'scripts') + if(threshold == 'DEBUG'): #FIXME TODO remove this on PROD + basedir = os.path.join(env.config.basedir, 'scripts', 'balancer-emulator') + command = ['python','hdfs-command.py'] + + _print("Executing command %s\n" % command) + + parser = hdfs_rebalance.HdfsParser() + proc = subprocess.Popen( + command, + stdout=subprocess.PIPE, + shell=False, + close_fds=True, + cwd=basedir + ) + for line in iter(proc.stdout.readline, ''): + _print('[balancer] %s %s' % (str(datetime.now()), line )) + pl = parser.parseLine(line) + if pl: + res = pl.toJson() + res['completePercent'] = calculateCompletePercent(parser.initialLine, pl) + + self.put_structured_out(res) + elif parser.state == 'PROCESS_FINISED' : + _print('[balancer] %s %s' % (str(datetime.now()), 'Process is finished' )) + self.put_structured_out({'completePercent' : 1}) + break + + proc.stdout.close() + proc.wait() + if proc.returncode != None and proc.returncode != 0: + raise Fail('Hdfs rebalance process exited with error. See the log output') + +def _print(line): + sys.stdout.write(line) + sys.stdout.flush() + +if __name__ == "__main__": + NameNode().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/params.py new file mode 100644 index 0000000..ff61dd7 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/params.py @@ -0,0 +1,235 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +import status_params +import os + +config = Script.get_config() +tmp_dir = Script.get_tmp_dir() + +#RPM versioning support +rpm_version = default("/configurations/cluster-env/rpm_version", None) + +#hadoop params +if rpm_version: + mapreduce_libs_path = "/usr/phd/current/hadoop-mapreduce-client/*" + hadoop_libexec_dir = "/usr/phd/current/hadoop-client/libexec" + hadoop_bin = "/usr/phd/current/hadoop-client/sbin" + hadoop_bin_dir = "/usr/phd/current/hadoop-client/bin" +else: + mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*" + hadoop_libexec_dir = "/usr/lib/hadoop/libexec" + hadoop_bin = "/usr/lib/hadoop/sbin" + hadoop_bin_dir = "/usr/bin" + +hadoop_conf_dir = "/etc/hadoop/conf" +hadoop_conf_empty_dir = "/etc/hadoop/conf.empty" +limits_conf_dir = "/etc/security/limits.d" + +execute_path = os.environ['PATH'] + os.pathsep + hadoop_bin_dir +ulimit_cmd = "ulimit -c unlimited; " + +#security params +security_enabled = config['configurations']['cluster-env']['security_enabled'] +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] +hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] +falcon_user = config['configurations']['falcon-env']['falcon_user'] + +#exclude file +hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", []) +exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude'] +update_exclude_file_only = config['commandParams']['update_exclude_file_only'] + +kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) +#hosts +hostname = config["hostname"] +rm_host = default("/clusterHostInfo/rm_host", []) +slave_hosts = default("/clusterHostInfo/slave_hosts", []) +hagios_server_hosts = default("/clusterHostInfo/nagios_server_host", []) +oozie_servers = default("/clusterHostInfo/oozie_server", []) +hcat_server_hosts = default("/clusterHostInfo/webhcat_server_host", []) +hive_server_host = default("/clusterHostInfo/hive_server_host", []) +hbase_master_hosts = default("/clusterHostInfo/hbase_master_hosts", []) +hs_host = default("/clusterHostInfo/hs_host", []) +jtnode_host = default("/clusterHostInfo/jtnode_host", []) +namenode_host = default("/clusterHostInfo/namenode_host", []) +nm_host = default("/clusterHostInfo/nm_host", []) +ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", []) +journalnode_hosts = default("/clusterHostInfo/journalnode_hosts", []) +zkfc_hosts = default("/clusterHostInfo/zkfc_hosts", []) +falcon_host = default("/clusterHostInfo/falcon_server_hosts", []) + +has_ganglia_server = not len(ganglia_server_hosts) == 0 +has_namenodes = not len(namenode_host) == 0 +has_jobtracker = not len(jtnode_host) == 0 +has_resourcemanager = not len(rm_host) == 0 +has_histroryserver = not len(hs_host) == 0 +has_hbase_masters = not len(hbase_master_hosts) == 0 +has_slaves = not len(slave_hosts) == 0 +has_nagios = not len(hagios_server_hosts) == 0 +has_oozie_server = not len(oozie_servers) == 0 +has_hcat_server_host = not len(hcat_server_hosts) == 0 +has_hive_server_host = not len(hive_server_host) == 0 +has_journalnode_hosts = not len(journalnode_hosts) == 0 +has_zkfc_hosts = not len(zkfc_hosts) == 0 +has_falcon_host = not len(falcon_host) == 0 + + +is_namenode_master = hostname in namenode_host +is_jtnode_master = hostname in jtnode_host +is_rmnode_master = hostname in rm_host +is_hsnode_master = hostname in hs_host +is_hbase_master = hostname in hbase_master_hosts +is_slave = hostname in slave_hosts + +if has_ganglia_server: + ganglia_server_host = ganglia_server_hosts[0] + +#users and groups +yarn_user = config['configurations']['yarn-env']['yarn_user'] +hbase_user = config['configurations']['hbase-env']['hbase_user'] +nagios_user = config['configurations']['nagios-env']['nagios_user'] +oozie_user = config['configurations']['oozie-env']['oozie_user'] +webhcat_user = config['configurations']['hive-env']['hcat_user'] +hcat_user = config['configurations']['hive-env']['hcat_user'] +hive_user = config['configurations']['hive-env']['hive_user'] +smoke_user = config['configurations']['cluster-env']['smokeuser'] +mapred_user = config['configurations']['mapred-env']['mapred_user'] +hdfs_user = status_params.hdfs_user +hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] + +user_group = config['configurations']['cluster-env']['user_group'] +proxyuser_group = config['configurations']['hadoop-env']['proxyuser_group'] +nagios_group = config['configurations']['nagios-env']['nagios_group'] + +#hadoop params +hadoop_pid_dir_prefix = status_params.hadoop_pid_dir_prefix + +hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix'] +hadoop_root_logger = config['configurations']['hadoop-env']['hadoop_root_logger'] + +dfs_domain_socket_path = config['configurations']['hdfs-site']['dfs.domain.socket.path'] +dfs_domain_socket_dir = os.path.dirname(dfs_domain_socket_path) + +jn_edits_dir = config['configurations']['hdfs-site']['dfs.journalnode.edits.dir'] + +dfs_name_dir = config['configurations']['hdfs-site']['dfs.namenode.name.dir'] + +namenode_dirs_created_stub_dir = format("{hdfs_log_dir_prefix}/{hdfs_user}") +namenode_dirs_stub_filename = "namenode_dirs_created" + +smoke_hdfs_user_dir = format("/user/{smoke_user}") +smoke_hdfs_user_mode = 0770 + +namenode_formatted_old_mark_dir = format("{hadoop_pid_dir_prefix}/hdfs/namenode/formatted/") +namenode_formatted_mark_dir = format("/var/lib/hdfs/namenode/formatted/") + +fs_checkpoint_dir = config['configurations']['hdfs-site']['dfs.namenode.checkpoint.dir'] + +dfs_data_dir = config['configurations']['hdfs-site']['dfs.datanode.data.dir'] +data_dir_mount_file = config['configurations']['hadoop-env']['dfs.datanode.data.dir.mount.file'] + +dfs_dn_addr = default('/configurations/hdfs-site/dfs.datanode.address', None) +dfs_dn_http_addr = default('/configurations/hdfs-site/dfs.datanode.http.address', None) +dfs_dn_https_addr = default('/configurations/hdfs-site/dfs.datanode.https.address', None) +dfs_http_policy = default('/configurations/hdfs-site/dfs.http.policy', None) + +# HDFS High Availability properties +dfs_ha_enabled = False +dfs_ha_nameservices = default("/configurations/hdfs-site/dfs.nameservices", None) +dfs_ha_namenode_ids = default(format("/configurations/hdfs-site/dfs.ha.namenodes.{dfs_ha_nameservices}"), None) + +namenode_id = None +namenode_rpc = None + +if dfs_ha_namenode_ids: + dfs_ha_namemodes_ids_list = dfs_ha_namenode_ids.split(",") + dfs_ha_namenode_ids_array_len = len(dfs_ha_namemodes_ids_list) + if dfs_ha_namenode_ids_array_len > 1: + dfs_ha_enabled = True +if dfs_ha_enabled: + for nn_id in dfs_ha_namemodes_ids_list: + nn_host = config['configurations']['hdfs-site'][format('dfs.namenode.rpc-address.{dfs_ha_nameservices}.{nn_id}')] + if hostname in nn_host: + namenode_id = nn_id + namenode_rpc = nn_host + +journalnode_address = default('/configurations/hdfs-site/dfs.journalnode.http-address', None) +if journalnode_address: + journalnode_port = journalnode_address.split(":")[1] + + +if security_enabled: + _dn_principal_name = config['configurations']['hdfs-site']['dfs.datanode.kerberos.principal'] + _dn_keytab = config['configurations']['hdfs-site']['dfs.datanode.keytab.file'] + _dn_principal_name = _dn_principal_name.replace('_HOST',hostname.lower()) + + dn_kinit_cmd = format("{kinit_path_local} -kt {_dn_keytab} {_dn_principal_name};") +else: + dn_kinit_cmd = "" + +import functools +#create partial functions with common arguments for every HdfsDirectory call +#to create hdfs directory we need to call params.HdfsDirectory in code +HdfsDirectory = functools.partial( + HdfsDirectory, + conf_dir=hadoop_conf_dir, + hdfs_user=hdfs_user, + security_enabled = security_enabled, + keytab = hdfs_user_keytab, + kinit_path_local = kinit_path_local, + bin_dir = hadoop_bin_dir +) + +io_compression_codecs = config['configurations']['core-site']['io.compression.codecs'] +if not "com.hadoop.compression.lzo" in io_compression_codecs: + exclude_packages = ["lzo", "hadoop-lzo", "hadoop-lzo-native", "liblzo2-2"] +else: + exclude_packages = [] +name_node_params = default("/commandParams/namenode", None) + +#hadoop params +hadoop_env_sh_template = config['configurations']['hadoop-env']['content'] + +#hadoop-env.sh +java_home = config['hostLevelParams']['java_home'] +stack_version = str(config['hostLevelParams']['stack_version']) + +stack_is_champlain_or_further = not (stack_version.startswith('2.0') or stack_version.startswith('2.1')) + +if stack_version.startswith('2.0') and System.get_instance().os_family != "suse": + # deprecated rhel jsvc_path + jsvc_path = "/usr/libexec/phd-utils" +else: + jsvc_path = "/usr/lib/phd-utils" + +hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize'] +namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize'] +namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize'] +namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize'] + +jtnode_opt_newsize = "200m" +jtnode_opt_maxnewsize = "200m" +jtnode_heapsize = "1024m" +ttnode_heapsize = "1024m" + +dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize'] +mapred_pid_dir_prefix = default("/configurations/mapred-env/mapred_pid_dir_prefix","/var/run/hadoop-mapreduce") +mapred_log_dir_prefix = default("/configurations/mapred-env/mapred_log_dir_prefix","/var/log/hadoop-mapreduce") http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/service_check.py new file mode 100644 index 0000000..36e26d6 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/service_check.py @@ -0,0 +1,120 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * + + +class HdfsServiceCheck(Script): + def service_check(self, env): + import params + + env.set_params(params) + unique = functions.get_unique_id_and_date() + dir = '/tmp' + tmp_file = format("{dir}/{unique}") + + safemode_command = "dfsadmin -safemode get | grep OFF" + + create_dir_cmd = format("fs -mkdir {dir}") + chmod_command = format("fs -chmod 777 {dir}") + test_dir_exists = format("su -s /bin/bash - {smoke_user} -c '{hadoop_bin_dir}/hadoop --config {hadoop_conf_dir} fs -test -e {dir}'") + cleanup_cmd = format("fs -rm {tmp_file}") + #cleanup put below to handle retries; if retrying there wil be a stale file + #that needs cleanup; exit code is fn of second command + create_file_cmd = format( + "{cleanup_cmd}; hadoop --config {hadoop_conf_dir} fs -put /etc/passwd {tmp_file}") + test_cmd = format("fs -test -e {tmp_file}") + if params.security_enabled: + Execute(format( + "su -s /bin/bash - {smoke_user} -c '{kinit_path_local} -kt {smoke_user_keytab} " + "{smoke_user}'")) + ExecuteHadoop(safemode_command, + user=params.smoke_user, + logoutput=True, + conf_dir=params.hadoop_conf_dir, + try_sleep=3, + tries=20, + bin_dir=params.hadoop_bin_dir + ) + ExecuteHadoop(create_dir_cmd, + user=params.smoke_user, + logoutput=True, + not_if=test_dir_exists, + conf_dir=params.hadoop_conf_dir, + try_sleep=3, + tries=5, + bin_dir=params.hadoop_bin_dir + ) + ExecuteHadoop(chmod_command, + user=params.smoke_user, + logoutput=True, + conf_dir=params.hadoop_conf_dir, + try_sleep=3, + tries=5, + bin_dir=params.hadoop_bin_dir + ) + ExecuteHadoop(create_file_cmd, + user=params.smoke_user, + logoutput=True, + conf_dir=params.hadoop_conf_dir, + try_sleep=3, + tries=5, + bin_dir=params.hadoop_bin_dir + ) + ExecuteHadoop(test_cmd, + user=params.smoke_user, + logoutput=True, + conf_dir=params.hadoop_conf_dir, + try_sleep=3, + tries=5, + bin_dir=params.hadoop_bin_dir + ) + if params.has_journalnode_hosts: + journalnode_port = params.journalnode_port + smoke_test_user = params.smoke_user + checkWebUIFileName = "checkWebUI.py" + checkWebUIFilePath = format("{tmp_dir}/{checkWebUIFileName}") + comma_sep_jn_hosts = ",".join(params.journalnode_hosts) + checkWebUICmd = format( + "su -s /bin/bash - {smoke_test_user} -c 'python {checkWebUIFilePath} -m " + "{comma_sep_jn_hosts} -p {journalnode_port}'") + File(checkWebUIFilePath, + content=StaticFile(checkWebUIFileName)) + + Execute(checkWebUICmd, + logoutput=True, + try_sleep=3, + tries=5 + ) + + if params.is_namenode_master: + if params.has_zkfc_hosts: + pid_dir = format("{hadoop_pid_dir_prefix}/{hdfs_user}") + pid_file = format("{pid_dir}/hadoop-{hdfs_user}-zkfc.pid") + check_zkfc_process_cmd = format( + "ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1") + Execute(check_zkfc_process_cmd, + logoutput=True, + try_sleep=3, + tries=5 + ) + + +if __name__ == "__main__": + HdfsServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/snamenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/snamenode.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/snamenode.py new file mode 100644 index 0000000..5eb25d2 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/snamenode.py @@ -0,0 +1,65 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +from hdfs_snamenode import snamenode +from hdfs import hdfs + + +class SNameNode(Script): + def install(self, env): + import params + + env.set_params(params) + + self.install_packages(env, params.exclude_packages) + + + def start(self, env): + import params + + env.set_params(params) + + self.configure(env) + snamenode(action="start") + + def stop(self, env): + import params + + env.set_params(params) + + snamenode(action="stop") + + def configure(self, env): + import params + + env.set_params(params) + hdfs() + snamenode(action="configure") + + def status(self, env): + import status_params + + env.set_params(status_params) + + check_process_status(status_params.snamenode_pid_file) + + +if __name__ == "__main__": + SNameNode().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/status_params.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/status_params.py new file mode 100644 index 0000000..1dd4750 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/status_params.py @@ -0,0 +1,31 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * + +config = Script.get_config() + +hadoop_pid_dir_prefix = config['configurations']['hadoop-env']['hadoop_pid_dir_prefix'] +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] +phd_pid_dir = format("{hadoop_pid_dir_prefix}/{hdfs_user}") +datanode_pid_file = format("{phd_pid_dir}/hadoop-{hdfs_user}-datanode.pid") +namenode_pid_file = format("{phd_pid_dir}/hadoop-{hdfs_user}-namenode.pid") +snamenode_pid_file = format("{phd_pid_dir}/hadoop-{hdfs_user}-secondarynamenode.pid") +journalnode_pid_file = format("{phd_pid_dir}/hadoop-{hdfs_user}-journalnode.pid") +zkfc_pid_file = format("{phd_pid_dir}/hadoop-{hdfs_user}-zkfc.pid") http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/utils.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/utils.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/utils.py new file mode 100644 index 0000000..5e1221a --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/utils.py @@ -0,0 +1,149 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +import os + +from resource_management import * +import re + + +def service(action=None, name=None, user=None, create_pid_dir=False, + create_log_dir=False): + import params + + pid_dir = format("{hadoop_pid_dir_prefix}/{user}") + pid_file = format("{pid_dir}/hadoop-{user}-{name}.pid") + log_dir = format("{hdfs_log_dir_prefix}/{user}") + check_process = format( + "ls {pid_file} >/dev/null 2>&1 &&" + " ps `cat {pid_file}` >/dev/null 2>&1") + + if create_pid_dir: + Directory(pid_dir, + owner=user, + recursive=True) + if create_log_dir: + Directory(log_dir, + owner=user, + recursive=True) + + hadoop_env_exports = { + 'HADOOP_LIBEXEC_DIR': params.hadoop_libexec_dir + } + + if params.security_enabled and name == "datanode": + dfs_dn_port = get_port(params.dfs_dn_addr) + dfs_dn_http_port = get_port(params.dfs_dn_http_addr) + dfs_dn_https_port = get_port(params.dfs_dn_https_addr) + + # We try to avoid inability to start datanode as a plain user due to usage of root-owned ports + if params.dfs_http_policy == "HTTPS_ONLY": + secure_ports_are_in_use = is_secure_port(dfs_dn_port) or is_secure_port(dfs_dn_https_port) + elif params.dfs_http_policy == "HTTP_AND_HTTPS": + secure_ports_are_in_use = is_secure_port(dfs_dn_port) or is_secure_port(dfs_dn_http_port) or is_secure_port(dfs_dn_https_port) + else: # params.dfs_http_policy == "HTTP_ONLY" or not defined: + secure_ports_are_in_use = is_secure_port(dfs_dn_port) or is_secure_port(dfs_dn_http_port) + + # Calculate HADOOP_SECURE_DN_* env vars, but not append them yet + # These variables should not be set when starting secure datanode as a non-root + ## On secure datanodes, user to run the datanode as after dropping privileges + hadoop_secure_dn_user = params.hdfs_user + ## Where log files are stored in the secure data environment. + hadoop_secure_dn_log_dir = format("{hdfs_log_dir_prefix}/{hadoop_secure_dn_user}") + ## The directory where pid files are stored in the secure data environment. + hadoop_secure_dn_pid_dir = format("{hadoop_pid_dir_prefix}/{hadoop_secure_dn_user}") + hadoop_secure_dn_exports = { + 'HADOOP_SECURE_DN_USER' : hadoop_secure_dn_user, + 'HADOOP_SECURE_DN_LOG_DIR' : hadoop_secure_dn_log_dir, + 'HADOOP_SECURE_DN_PID_DIR' : hadoop_secure_dn_pid_dir + } + hadoop_secure_dn_pid_file = format("{hadoop_secure_dn_pid_dir}/hadoop_secure_dn.pid") + + # At Champlain stack and further, we may start datanode as a non-root even in secure cluster + if not params.stack_is_champlain_or_further or secure_ports_are_in_use: + user = "root" + pid_file = format( + "{hadoop_pid_dir_prefix}/{hdfs_user}/hadoop-{hdfs_user}-{name}.pid") + if params.stack_is_champlain_or_further: + hadoop_env_exports.update(hadoop_secure_dn_exports) + + if action == 'stop' and params.stack_is_champlain_or_further and \ + os.path.isfile(hadoop_secure_dn_pid_file): + # We need special handling for this case to handle the situation + # when we configure non-root secure DN and then restart it + # to handle new configs. Otherwise we will not be able to stop + # a running instance + user = "root" + try: + with open(hadoop_secure_dn_pid_file, 'r') as f: + pid = f.read() + os.kill(int(pid), 0) + hadoop_env_exports.update(hadoop_secure_dn_exports) + except IOError: + pass # Can not open pid file + except ValueError: + pass # Pid file content is invalid + except OSError: + pass # Process is not running + + + hadoop_env_exports_str = '' + for exp in hadoop_env_exports.items(): + hadoop_env_exports_str += "export {0}={1} && ".format(exp[0], exp[1]) + + hadoop_daemon = format( + "{hadoop_env_exports_str}" + "{hadoop_bin}/hadoop-daemon.sh") + cmd = format("{hadoop_daemon} --config {hadoop_conf_dir}") + + daemon_cmd = format("{ulimit_cmd} su -s /bin/bash - {user} -c '{cmd} {action} {name}'") + + service_is_up = check_process if action == "start" else None + #remove pid file from dead process + File(pid_file, + action="delete", + not_if=check_process, + ) + Execute(daemon_cmd, + not_if=service_is_up + ) + if action == "stop": + File(pid_file, + action="delete", + ) + +def get_port(address): + """ + Extracts port from the address like 0.0.0.0:1019 + """ + if address is None: + return None + m = re.search(r'(?:http(?:s)?://)?([\w\d.]*):(\d{1,5})', address) + if m is not None: + return int(m.group(2)) + else: + return None + +def is_secure_port(port): + """ + Returns True if port is root-owned at *nix systems + """ + if port is not None: + return port < 1024 + else: + return False \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/zkfc_slave.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/zkfc_slave.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/zkfc_slave.py new file mode 100644 index 0000000..fd9bbfa --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/zkfc_slave.py @@ -0,0 +1,64 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +from utils import service +from hdfs import hdfs + + +class ZkfcSlave(Script): + def install(self, env): + import params + + self.install_packages(env, params.exclude_packages) + env.set_params(params) + + def start(self, env): + import params + + env.set_params(params) + self.configure(env) + service( + action="start", name="zkfc", user=params.hdfs_user, create_pid_dir=True, + create_log_dir=True + ) + + def stop(self, env): + import params + + env.set_params(params) + service( + action="stop", name="zkfc", user=params.hdfs_user, create_pid_dir=True, + create_log_dir=True + ) + + def configure(self, env): + hdfs() + pass + + def status(self, env): + import status_params + + env.set_params(status_params) + + check_process_status(status_params.zkfc_pid_file) + + +if __name__ == "__main__": + ZkfcSlave().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/templates/exclude_hosts_list.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/templates/exclude_hosts_list.j2 b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/templates/exclude_hosts_list.j2 new file mode 100644 index 0000000..a92cdc1 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/templates/exclude_hosts_list.j2 @@ -0,0 +1,21 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{% for host in hdfs_exclude_file %} +{{host}} +{% endfor %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/templates/hdfs.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/templates/hdfs.conf.j2 b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/templates/hdfs.conf.j2 new file mode 100644 index 0000000..d58a6f5 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/templates/hdfs.conf.j2 @@ -0,0 +1,35 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{hdfs_user}} - nofile 32768 +{{hdfs_user}} - nproc 65536 http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/templates/slaves.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/templates/slaves.j2 b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/templates/slaves.j2 new file mode 100644 index 0000000..4a9e713 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/templates/slaves.j2 @@ -0,0 +1,21 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{% for host in slave_hosts %} +{{host}} +{% endfor %} http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hcat-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hcat-env.xml b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hcat-env.xml new file mode 100644 index 0000000..91b402b --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hcat-env.xml @@ -0,0 +1,57 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + <!-- hcat-env.sh --> + <property> + <name>content</name> + <description>This is the jinja template for hcat-env.sh file</description> + <value> + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + JAVA_HOME={{java64_home}} + HCAT_PID_DIR={{hcat_pid_dir}}/ + HCAT_LOG_DIR={{hcat_log_dir}}/ + HCAT_CONF_DIR={{hcat_conf_dir}} + HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}} + #DBROOT is the path where the connector jars are downloaded + DBROOT={{hcat_dbroot}} + USER={{hcat_user}} + METASTORE_PORT={{hive_metastore_port}} + </value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hive-env.xml b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hive-env.xml new file mode 100644 index 0000000..7444331 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hive-env.xml @@ -0,0 +1,134 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + <property> + <name>hive_database_type</name> + <value>postgres</value> + <description>Default HIVE DB type.</description> + </property> + <property> + <name>hive_database</name> + <value>New PostgreSQL Database</value> + <description> + Property that determines whether the HIVE DB is managed by Ambari. + </description> + </property> + <property> + <name>hive_ambari_database</name> + <value>PostgreSQL</value> + <description>Database type.</description> + </property> + <property> + <name>hive_database_name</name> + <value>hive</value> + <description>Database name.</description> + </property> + <property> + <name>hive_dbroot</name> + <value>/usr/lib/hive/lib/</value> + <description>Hive DB Directory.</description> + </property> + <property> + <name>hive_log_dir</name> + <value>/var/log/hive</value> + <description>Directory for Hive Log files.</description> + </property> + <property> + <name>hive_pid_dir</name> + <value>/var/run/hive</value> + <description>Hive PID Dir.</description> + </property> + <property> + <name>hive_user</name> + <value>hive</value> + <property-type>USER</property-type> + <description>Hive User.</description> + </property> + + <!--HCAT--> + + <property> + <name>hcat_log_dir</name> + <value>/var/log/webhcat</value> + <description>WebHCat Log Dir.</description> + </property> + <property> + <name>hcat_pid_dir</name> + <value>/var/run/webhcat</value> + <description>WebHCat Pid Dir.</description> + </property> + <property> + <name>hcat_user</name> + <value>hcat</value> + <property-type>USER</property-type> + <description>HCat User.</description> + </property> + <property> + <name>webhcat_user</name> + <value>hcat</value> + <property-type>USER</property-type> + <description>WebHCat User.</description> + </property> + + <!-- hive-env.sh --> + <property> + <name>content</name> + <description>This is the jinja template for hive-env.sh file</description> + <value> + if [ "$SERVICE" = "cli" ]; then + if [ -z "$DEBUG" ]; then + export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseParNewGC -XX:-UseGCOverheadLimit" + else + export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit" + fi + fi + +# The heap size of the jvm stared by hive shell script can be controlled via: + +export HADOOP_HEAPSIZE="{{hive_heapsize}}" +export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS" + +# Larger heap size may be required when running queries over large number of files or partitions. +# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be +# appropriate for hive server (hwi etc). + + +# Set HADOOP_HOME to point to a specific hadoop install directory +HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}} + +# Hive Configuration Directory can be controlled by: +export HIVE_CONF_DIR={{hive_config_dir}} + +# Folder containing extra ibraries required for hive compilation/execution can be controlled by: +if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then + export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH} +elif [ -d "/usr/lib/hive-hcatalog/" ]; then + export HIVE_AUX_JARS_PATH=/usr/lib/hive-hcatalog/share/hcatalog/hive-hcatalog-core-*.jar +else + export HIVE_AUX_JARS_PATH=/usr/lib/hcatalog/share/hcatalog/hcatalog-core.jar +fi +export METASTORE_PORT={{hive_metastore_port}} + </value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hive-exec-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hive-exec-log4j.xml b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hive-exec-log4j.xml new file mode 100644 index 0000000..fb852f7 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hive-exec-log4j.xml @@ -0,0 +1,111 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration supports_final="false"> + + <property> + <name>content</name> + <description>Custom hive-exec-log4j</description> + <value> +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Define some default values that can be overridden by system properties + +hive.log.threshold=ALL +hive.root.logger=INFO,FA +hive.log.dir=${java.io.tmpdir}/${user.name} +hive.query.id=hadoop +hive.log.file=${hive.query.id}.log + +# Define the root logger to the system property "hadoop.root.logger". +log4j.rootLogger=${hive.root.logger}, EventCounter + +# Logging Threshold +log4j.threshhold=${hive.log.threshold} + +# +# File Appender +# + +log4j.appender.FA=org.apache.log4j.FileAppender +log4j.appender.FA.File=${hive.log.dir}/${hive.log.file} +log4j.appender.FA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +# Debugging Pattern format +log4j.appender.FA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + + +# +# console +# Add "console" to rootlogger above if you want to use this +# + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n + +#custom logging levels +#log4j.logger.xxx=DEBUG + +# +# Event Counter Appender +# Sends counts of logging messages at different severity levels to Hadoop Metrics. +# +log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter + + +log4j.category.DataNucleus=ERROR,FA +log4j.category.Datastore=ERROR,FA +log4j.category.Datastore.Schema=ERROR,FA +log4j.category.JPOX.Datastore=ERROR,FA +log4j.category.JPOX.Plugin=ERROR,FA +log4j.category.JPOX.MetaData=ERROR,FA +log4j.category.JPOX.Query=ERROR,FA +log4j.category.JPOX.General=ERROR,FA +log4j.category.JPOX.Enhancer=ERROR,FA + + +# Silence useless ZK logs +log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,FA +log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,FA + + </value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hive-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hive-log4j.xml b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hive-log4j.xml new file mode 100644 index 0000000..cb8d3a9 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hive-log4j.xml @@ -0,0 +1,120 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration supports_final="false"> + + <property> + <name>content</name> + <description>Custom log4j.properties</description> + <value> +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Define some default values that can be overridden by system properties +hive.log.threshold=ALL +hive.root.logger=INFO,DRFA +hive.log.dir=${java.io.tmpdir}/${user.name} +hive.log.file=hive.log + +# Define the root logger to the system property "hadoop.root.logger". +log4j.rootLogger=${hive.root.logger}, EventCounter + +# Logging Threshold +log4j.threshold=${hive.log.threshold} + +# +# Daily Rolling File Appender +# +# Use the PidDailyerRollingFileAppend class instead if you want to use separate log files +# for different CLI session. +# +# log4j.appender.DRFA=org.apache.hadoop.hive.ql.log.PidDailyRollingFileAppender + +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender + +log4j.appender.DRFA.File=${hive.log.dir}/${hive.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +# Debugging Pattern format +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + + +# +# console +# Add "console" to rootlogger above if you want to use this +# + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n +log4j.appender.console.encoding=UTF-8 + +#custom logging levels +#log4j.logger.xxx=DEBUG + +# +# Event Counter Appender +# Sends counts of logging messages at different severity levels to Hadoop Metrics. +# +log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter + + +log4j.category.DataNucleus=ERROR,DRFA +log4j.category.Datastore=ERROR,DRFA +log4j.category.Datastore.Schema=ERROR,DRFA +log4j.category.JPOX.Datastore=ERROR,DRFA +log4j.category.JPOX.Plugin=ERROR,DRFA +log4j.category.JPOX.MetaData=ERROR,DRFA +log4j.category.JPOX.Query=ERROR,DRFA +log4j.category.JPOX.General=ERROR,DRFA +log4j.category.JPOX.Enhancer=ERROR,DRFA + + +# Silence useless ZK logs +log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,DRFA +log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,DRFA + </value> + </property> + +</configuration>
