Repository: ambari Updated Branches: refs/heads/branch-2.1 deb5a06a4 -> 989d34c14 refs/heads/trunk f0949259f -> 9286b1c02
AMBARI-13290. Set kafka userid ulimit open files to 32k for kafka broker through Ambari (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9286b1c0 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9286b1c0 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9286b1c0 Branch: refs/heads/trunk Commit: 9286b1c0206221ffd4fe0774c1b151da12ac8cc3 Parents: f094925 Author: Andrew Onishuk <aonis...@hortonworks.com> Authored: Fri Oct 2 14:41:30 2015 +0300 Committer: Andrew Onishuk <aonis...@hortonworks.com> Committed: Fri Oct 2 14:41:30 2015 +0300 ---------------------------------------------------------------------- .../KAFKA/0.8.1.2.2/configuration/kafka-env.xml | 10 ++++++ .../KAFKA/0.8.1.2.2/package/scripts/kafka.py | 13 ++++++++ .../KAFKA/0.8.1.2.2/package/scripts/params.py | 4 +++ .../0.8.1.2.2/package/templates/kafka.conf.j2 | 35 ++++++++++++++++++++ 4 files changed, 62 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9286b1c0/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/configuration/kafka-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/configuration/kafka-env.xml b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/configuration/kafka-env.xml index ecc0782..d515e73 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/configuration/kafka-env.xml +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/configuration/kafka-env.xml @@ -42,6 +42,16 @@ <overridable>false</overridable> </value-attributes> </property> + <property> + <name>kafka_user_nofile_limit</name> + <value>32000</value> + <description>Max open files limit setting for KAFKA user.</description> + </property> + <property> + <name>kafka_user_nproc_limit</name> + <value>16000</value> + <description>Max number of processes limit setting for KAFKA user.</description> + </property> <!-- kafka-env.sh --> <property> http://git-wip-us.apache.org/repos/asf/ambari/blob/9286b1c0/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka.py index 11492a7..4d28c41 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka.py @@ -94,6 +94,19 @@ def kafka(): TemplateConfig(format("{conf_dir}/kafka_client_jaas.conf"), owner=params.kafka_user) + # On some OS this folder could be not exists, so we will create it before pushing there files + Directory(params.limits_conf_dir, + recursive=True, + owner='root', + group='root' + ) + + File(os.path.join(params.limits_conf_dir, 'kafka.conf'), + owner='root', + group='root', + mode=0644, + content=Template("kafka.conf.j2") + ) setup_symlink(params.kafka_managed_pid_dir, params.kafka_pid_dir) setup_symlink(params.kafka_managed_log_dir, params.kafka_log_dir) http://git-wip-us.apache.org/repos/asf/ambari/blob/9286b1c0/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py index 26ea3e2..0a55504 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py @@ -43,6 +43,10 @@ hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) kafka_home = '/usr/lib/kafka/' kafka_bin = kafka_home+'/bin/kafka' conf_dir = "/etc/kafka/conf" +limits_conf_dir = "/etc/security/limits.d" + +kafka_user_nofile_limit = config['configurations']['kafka-env']['kafka_user_nofile_limit'] +kafka_user_nproc_limit = config['configurations']['kafka-env']['kafka_user_nproc_limit'] # parameters for 2.2+ if Script.is_hdp_stack_greater_or_equal("2.2"): http://git-wip-us.apache.org/repos/asf/ambari/blob/9286b1c0/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/templates/kafka.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/templates/kafka.conf.j2 b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/templates/kafka.conf.j2 new file mode 100644 index 0000000..9e18e1d --- /dev/null +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/templates/kafka.conf.j2 @@ -0,0 +1,35 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{kafka_user}} - nofile {{kafka_user_nofile_limit}} +{{kafka_user}} - nproc {{kafka_user_nproc_limit}}