Repository: ambari Updated Branches: refs/heads/branch-1.7.0 f906af1ac -> 21d965ef1
AMBARI-8074. Provide validated Oozie configs for Ambari deployments. (swagle) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/21d965ef Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/21d965ef Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/21d965ef Branch: refs/heads/branch-1.7.0 Commit: 21d965ef18f2f11c375d46da3cea83b4d9b6670d Parents: f906af1 Author: Siddharth Wagle <swa...@hortonworks.com> Authored: Fri Oct 31 15:29:43 2014 -0700 Committer: Siddharth Wagle <swa...@hortonworks.com> Committed: Fri Oct 31 15:29:53 2014 -0700 ---------------------------------------------------------------------- .../services/OOZIE/package/scripts/oozie.py | 43 ++++++-- .../services/OOZIE/package/scripts/params.py | 1 + .../services/OOZIE/configuration/oozie-env.xml | 101 +++++++++++++++++++ .../stacks/2.0.6/OOZIE/test_oozie_client.py | 1 + .../stacks/2.0.6/OOZIE/test_oozie_server.py | 37 +++++++ 5 files changed, 174 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/21d965ef/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/oozie.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/oozie.py index 19c334c..f33fe81 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/oozie.py @@ -31,12 +31,12 @@ def oozie(is_server=False # TODO: see if see can remove this owner=params.oozie_user, mode=params.oozie_hdfs_user_mode ) - Directory( params.conf_dir, + Directory(params.conf_dir, recursive = True, owner = params.oozie_user, group = params.user_group ) - XmlConfig( "oozie-site.xml", + XmlConfig("oozie-site.xml", conf_dir = params.conf_dir, configurations = params.config['configurations']['oozie-site'], configuration_attributes=params.config['configuration_attributes']['oozie-site'], @@ -90,14 +90,14 @@ def oozie(is_server=False # TODO: see if see can remove this not_if = format("[ -f {check_db_connection_jar} ]"), environment=environment ) - - oozie_ownership( ) + pass + + oozie_ownership() if is_server: - oozie_server_specific( ) + oozie_server_specific() -def oozie_ownership( -): +def oozie_ownership(): import params File ( format("{conf_dir}/hadoop-config.xml"), @@ -120,8 +120,7 @@ def oozie_ownership( group = params.user_group ) -def oozie_server_specific( -): +def oozie_server_specific(): import params File(params.pid_file, @@ -157,4 +156,30 @@ def oozie_server_specific( user = params.oozie_user, not_if = no_op_test ) + + if params.stack_is_hdp22_or_further: + # Create hive-site and tez-site configs for oozie + Directory(params.hive_conf_dir, + recursive = True, + owner = params.oozie_user, + group = params.user_group + ) + if params.config['configurations']['hive-site']: + XmlConfig("hive-site.xml", + conf_dir=params.hive_conf_dir, + configurations=params.config['configurations']['hive-site'], + configuration_attributes=params.config['configuration_attributes']['hive-site'], + owner=params.oozie_user, + group=params.user_group, + mode=0644) + if params.config['configurations']['tez-site']: + XmlConfig( "tez-site.xml", + conf_dir = params.hive_conf_dir, + configurations = params.config['configurations']['tez-site'], + configuration_attributes=params.config['configuration_attributes']['tez-site'], + owner = params.oozie_user, + group = params.user_group, + mode = 0664 + ) + pass http://git-wip-us.apache.org/repos/asf/ambari/blob/21d965ef/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/params.py index 6768014..fbe032d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/params.py @@ -63,6 +63,7 @@ execute_path = oozie_bin_dir + os.pathsep + hadoop_bin_dir hadoop_conf_dir = "/etc/hadoop/conf" conf_dir = "/etc/oozie/conf" +hive_conf_dir = "/etc/oozie/conf/action-conf/hive" oozie_user = config['configurations']['oozie-env']['oozie_user'] smokeuser = config['configurations']['cluster-env']['smokeuser'] user_group = config['configurations']['cluster-env']['user_group'] http://git-wip-us.apache.org/repos/asf/ambari/blob/21d965ef/ambari-server/src/main/resources/stacks/HDP/2.2/services/OOZIE/configuration/oozie-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/OOZIE/configuration/oozie-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/OOZIE/configuration/oozie-env.xml new file mode 100644 index 0000000..39ba798 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/OOZIE/configuration/oozie-env.xml @@ -0,0 +1,101 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + <!-- oozie-env.sh --> + <property> + <name>content</name> + <description>This is the jinja template for oozie-env.sh file</description> + <value> +#!/bin/bash + +if [ -d "/usr/lib/bigtop-tomcat" ]; then + export OOZIE_CONFIG=${OOZIE_CONFIG:-/etc/oozie/conf} + export CATALINA_BASE=${CATALINA_BASE:-{{oozie_server_dir}}} + export CATALINA_TMPDIR=${CATALINA_TMPDIR:-/var/tmp/oozie} + export OOZIE_CATALINA_HOME=/usr/lib/bigtop-tomcat +fi + +#Set JAVA HOME +export JAVA_HOME={{java_home}} + +export JRE_HOME=${JAVA_HOME} + +# Set Oozie specific environment variables here. + +# Settings for the Embedded Tomcat that runs Oozie +# Java System properties for Oozie should be specified in this variable +# +# export CATALINA_OPTS= + +# Oozie configuration file to load from Oozie configuration directory +# +# export OOZIE_CONFIG_FILE=oozie-site.xml + +# Oozie logs directory +# +export OOZIE_LOG={{oozie_log_dir}} + +# Oozie pid directory +# +export CATALINA_PID={{pid_file}} + +#Location of the data for oozie +export OOZIE_DATA={{oozie_data_dir}} + +# Oozie Log4J configuration file to load from Oozie configuration directory +# +# export OOZIE_LOG4J_FILE=oozie-log4j.properties + +# Reload interval of the Log4J configuration file, in seconds +# +# export OOZIE_LOG4J_RELOAD=10 + +# The port Oozie server runs +# +export OOZIE_HTTP_PORT={{oozie_server_port}} + +# The admin port Oozie server runs +# +export OOZIE_ADMIN_PORT={{oozie_server_admin_port}} + +# The host name Oozie server runs on +# +# export OOZIE_HTTP_HOSTNAME=`hostname -f` + +# The base URL for callback URLs to Oozie +# +# export OOZIE_BASE_URL="http://${OOZIE_HTTP_HOSTNAME}:${OOZIE_HTTP_PORT}/oozie"; +export JAVA_LIBRARY_PATH={{hadoop_lib_home}}/native/Linux-amd64-64 + +# At least 1 minute of retry time to account for server downtime during +# upgrade/downgrade +export OOZIE_CLIENT_OPTS="${OOZIE_CLIENT_OPTS} -Doozie.connection.retry.count=5 " + +# This is needed so that Oozie does not run into OOM or GC Overhead limit +# exceeded exceptions. If the oozie server is handling large number of +# workflows/coordinator jobs, the memory settings may need to be revised +export CATALINA_OPTS="${CATALINA_OPTS} -Xmx2048m -XX:MaxPermSize=256m " + </value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/21d965ef/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_client.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_client.py index 67482b4..c19c312 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_client.py +++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_client.py @@ -126,6 +126,7 @@ class TestOozieClient(RMFTestCase): ) self.assertNoMoreResources() + def test_configure_default_hdp22(self): config_file = "stacks/2.0.6/configs/default.json" with open(config_file, "r") as f: http://git-wip-us.apache.org/repos/asf/ambari/blob/21d965ef/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py index d9f3ef6..9bf2085 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py @@ -343,3 +343,40 @@ class TestOozieServer(RMFTestCase): user = 'oozie', ) + def test_configure_default_hdp22(self): + config_file = "stacks/2.0.6/configs/default.json" + with open(config_file, "r") as f: + default_json = json.load(f) + + default_json['hostLevelParams']['stack_version']= '2.2' + self.executeScript("2.0.6/services/OOZIE/package/scripts/oozie_server.py", + classname = "OozieServer", + command = "configure", + config_file="default.json" + ) + self.assert_configure_default() + self.assertResourceCalled('Directory', '/etc/oozie/conf/action-conf/hive', + owner = 'oozie', + group = 'hadoop', + recursive = True + ) + self.assertResourceCalled('XmlConfig', 'hive-site', + owner = 'oozie', + group = 'hadoop', + mode = 0664, + conf_dir = '/etc/oozie/conf/action-conf/hive', + configurations = self.getConfig()['configurations']['hive-site'], + configuration_attributes = self.getConfig()['configuration_attributes']['hive-site'] + ) + self.assertResourceCalled('XmlConfig', 'tez-site', + owner = 'oozie', + group = 'hadoop', + mode = 0664, + conf_dir = '/etc/oozie/conf/action-conf/hive', + configurations = self.getConfig()['configurations']['tez-site'], + configuration_attributes = self.getConfig()['configuration_attributes']['tez-site'] + ) + self.assertNoMoreResources() + + +
