Repository: ambari Updated Branches: refs/heads/trunk 7e395b9bb -> b7dd31447
AMBARI-5720. pig.properties should set pig.location.check.strict to false. (Jonathan Hurley via swagle) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b7dd3144 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b7dd3144 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b7dd3144 Branch: refs/heads/trunk Commit: b7dd314479888fe88c6483e875e78396ecb933fe Parents: 7e395b9 Author: Siddharth Wagle <swa...@hortonworks.com> Authored: Mon May 12 10:15:55 2014 -0700 Committer: Siddharth Wagle <swa...@hortonworks.com> Committed: Mon May 12 10:15:55 2014 -0700 ---------------------------------------------------------------------- .../PIG/configuration/pig-properties.xml | 89 +++++++++++++++++++ .../services/PIG/configuration/pig.properties | 52 ----------- .../stacks/HDP/2.0.6/services/PIG/metainfo.xml | 1 + .../services/PIG/package/scripts/params.py | 11 ++- .../2.0.6/services/PIG/package/scripts/pig.py | 9 +- .../PIG/package/templates/pig.properties.j2 | 73 ---------------- .../PIG/configuration/pig-properties.xml | 91 ++++++++++++++++++++ .../python/stacks/2.0.6/PIG/test_pig_client.py | 18 ++-- .../python/stacks/2.0.6/configs/default.json | 8 +- .../python/stacks/2.0.6/configs/secured.json | 3 + 10 files changed, 221 insertions(+), 134 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b7dd3144/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig-properties.xml new file mode 100644 index 0000000..185d3b0 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig-properties.xml @@ -0,0 +1,89 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + + <property> + <name>pig-content</name> + <description>Describe all the Pig agent configurations</description> + <value> +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Pig default configuration file. All values can be overwritten by pig.properties and command line arguments. +# see bin/pig -help + +# brief logging (no timestamps) +brief=false + +# debug level, INFO is default +debug=INFO + +# verbose print all log messages to screen (default to print only INFO and above to screen) +verbose=false + +# exectype local|mapreduce, mapreduce is default +exectype=mapreduce + +# Enable insertion of information about script into hadoop job conf +pig.script.info.enabled=true + +# Do not spill temp files smaller than this size (bytes) +pig.spill.size.threshold=5000000 + +# EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes) +# This should help reduce the number of files being spilled. +pig.spill.gc.activation.size=40000000 + +# the following two parameters are to help estimate the reducer number +pig.exec.reducers.bytes.per.reducer=1000000000 +pig.exec.reducers.max=999 + +# Temporary location to store the intermediate data. +pig.temp.dir=/tmp/ + +# Threshold for merging FRJoin fragment files +pig.files.concatenation.threshold=100 +pig.optimistic.files.concatenation=false; + +pig.disable.counter=false + +hcat.bin=/usr/bin/hcat + + </value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/b7dd3144/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig.properties ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig.properties b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig.properties deleted file mode 100644 index 01000b5..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig.properties +++ /dev/null @@ -1,52 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Pig default configuration file. All values can be overwritten by pig.properties and command line arguments. -# see bin/pig -help - -# brief logging (no timestamps) -brief=false - -#debug level, INFO is default -debug=INFO - -#verbose print all log messages to screen (default to print only INFO and above to screen) -verbose=false - -#exectype local|mapreduce, mapreduce is default -exectype=mapreduce - -#Enable insertion of information about script into hadoop job conf -pig.script.info.enabled=true - -#Do not spill temp files smaller than this size (bytes) -pig.spill.size.threshold=5000000 -#EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes) -#This should help reduce the number of files being spilled. -pig.spill.gc.activation.size=40000000 - -#the following two parameters are to help estimate the reducer number -pig.exec.reducers.bytes.per.reducer=1000000000 -pig.exec.reducers.max=999 - -#Temporary location to store the intermediate data. -pig.temp.dir=/tmp/ - -#Threshold for merging FRJoin fragment files -pig.files.concatenation.threshold=100 -pig.optimistic.files.concatenation=false; - -pig.disable.counter=false http://git-wip-us.apache.org/repos/asf/ambari/blob/b7dd3144/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/metainfo.xml index 7db411b..c8a3cf4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/metainfo.xml @@ -55,6 +55,7 @@ <configuration-dependencies> <config-type>global</config-type> <config-type>pig-log4j</config-type> + <config-type>pig-properties</config-type> </configuration-dependencies> </service> http://git-wip-us.apache.org/repos/asf/ambari/blob/b7dd3144/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py index c44acfa..747687d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py @@ -38,7 +38,16 @@ kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), " java64_home = config['hostLevelParams']['java_home'] hadoop_home = "/usr" -#log4j.properties +# pig.properties - if not in the JSON command, then we need to esnure some +# basic properties are set; this is a safety mechanism +if (('pig-properties' in config['configurations']) and ('pig-content' in config['configurations']['pig-properties'])): + pig_properties = config['configurations']['pig-properties']['pig-content'] +else: + pig_properties = {} + pig_properties['hcat.bin'] = '/usr/bin/hcat' + pig_properties['pig.location.check.strict'] = 'false' + +# log4j.properties if (('pig-log4j' in config['configurations']) and ('content' in config['configurations']['pig-log4j'])): log4j_props = config['configurations']['pig-log4j']['content'] else: http://git-wip-us.apache.org/repos/asf/ambari/blob/b7dd3144/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/pig.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/pig.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/pig.py index 329a0bc..629f7cb 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/pig.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/pig.py @@ -30,7 +30,14 @@ def pig(): group = params.user_group ) - pig_TemplateConfig( ['pig-env.sh','pig.properties']) + pig_TemplateConfig( ['pig-env.sh']) + + # pig_properties is always set to a default even if it's not in the payload + PropertiesFile(format("{params.pig_conf_dir}/pig.properties"), + mode = 0644, + group=params.user_group, + owner=params.hdfs_user, + properties=params.pig_properties) if (params.log4j_props != None): File(format("{params.pig_conf_dir}/log4j.properties"), http://git-wip-us.apache.org/repos/asf/ambari/blob/b7dd3144/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/templates/pig.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/templates/pig.properties.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/templates/pig.properties.j2 deleted file mode 100644 index 90a32b1..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/templates/pig.properties.j2 +++ /dev/null @@ -1,73 +0,0 @@ -{# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# -# - -# Pig configuration file. All values can be overwritten by command line arguments. - -# log4jconf log4j configuration file -# log4jconf=./conf/log4j.properties - -# a file that contains pig script -#file= - -# load jarfile, colon separated -#jar= - -#verbose print all log messages to screen (default to print only INFO and above to screen) -#verbose=true - -#exectype local|mapreduce, mapreduce is default -#exectype=local - -#pig.logfile= - -#Do not spill temp files smaller than this size (bytes) -#pig.spill.size.threshold=5000000 -#EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes) -#This should help reduce the number of files being spilled. -#pig.spill.gc.activation.size=40000000 - -#the following two parameters are to help estimate the reducer number -#pig.exec.reducers.bytes.per.reducer=1000000000 -#pig.exec.reducers.max=999 - -#Use this option only when your Pig job will otherwise die because of -#using more counter than hadoop configured limit -#pig.disable.counter=true -hcat.bin=/usr/bin/hcat http://git-wip-us.apache.org/repos/asf/ambari/blob/b7dd3144/ambari-server/src/main/resources/stacks/HDP/2.1/services/PIG/configuration/pig-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/PIG/configuration/pig-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/services/PIG/configuration/pig-properties.xml new file mode 100644 index 0000000..97d42bb --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/PIG/configuration/pig-properties.xml @@ -0,0 +1,91 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + + <property> + <name>pig-content</name> + <value> +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Pig default configuration file. All values can be overwritten by pig.properties and command line arguments. +# see bin/pig -help + +# brief logging (no timestamps) +brief=false + +# debug level, INFO is default +debug=INFO + +# verbose print all log messages to screen (default to print only INFO and above to screen) +verbose=false + +# exectype local|mapreduce, mapreduce is default +exectype=mapreduce + +# Enable insertion of information about script into hadoop job conf +pig.script.info.enabled=true + +# Do not spill temp files smaller than this size (bytes) +pig.spill.size.threshold=5000000 + +# EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes) +# This should help reduce the number of files being spilled. +pig.spill.gc.activation.size=40000000 + +# the following two parameters are to help estimate the reducer number +pig.exec.reducers.bytes.per.reducer=1000000000 +pig.exec.reducers.max=999 + +# Temporary location to store the intermediate data. +pig.temp.dir=/tmp/ + +# Threshold for merging FRJoin fragment files +pig.files.concatenation.threshold=100 +pig.optimistic.files.concatenation=false; + +pig.disable.counter=false + +# Avoid pig failures when multiple jobs write to the same location +pig.location.check.strict=false + +hcat.bin=/usr/bin/hcat + + </value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/b7dd3144/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py b/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py index 5ff3b4f..87b8e0c 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py +++ b/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py @@ -31,13 +31,16 @@ class TestPigClient(RMFTestCase): self.assertResourceCalled('Directory', '/etc/pig/conf', owner = 'hdfs', - group = 'hadoop', + group = 'hadoop' ) self.assertResourceCalled('TemplateConfig', '/etc/pig/conf/pig-env.sh', - owner = 'hdfs', + owner = 'hdfs' ) - self.assertResourceCalled('TemplateConfig', '/etc/pig/conf/pig.properties', + self.assertResourceCalled('PropertiesFile', '/etc/pig/conf/pig.properties', owner = 'hdfs', + group = 'hadoop', + mode = 0644, + properties = 'pigproperties\nline2' ) self.assertResourceCalled('File', '/etc/pig/conf/log4j.properties', owner = 'hdfs', @@ -56,13 +59,16 @@ class TestPigClient(RMFTestCase): self.assertResourceCalled('Directory', '/etc/pig/conf', owner = 'hdfs', - group = 'hadoop', + group = 'hadoop' ) self.assertResourceCalled('TemplateConfig', '/etc/pig/conf/pig-env.sh', - owner = 'hdfs', + owner = 'hdfs' ) - self.assertResourceCalled('TemplateConfig', '/etc/pig/conf/pig.properties', + self.assertResourceCalled('PropertiesFile', '/etc/pig/conf/pig.properties', owner = 'hdfs', + group = 'hadoop', + mode = 0644, + properties = 'pigproperties\nline2' ) self.assertResourceCalled('File', '/etc/pig/conf/log4j.properties', owner = 'hdfs', http://git-wip-us.apache.org/repos/asf/ambari/blob/b7dd3144/ambari-server/src/test/python/stacks/2.0.6/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json index 3561b4b..dc61567 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json @@ -512,6 +512,9 @@ "pig-log4j": { "content": "log4jproperties\nline2" }, + "pig-properties": { + "pig-content": "pigproperties\nline2" + }, "oozie-log4j": { "content": "log4jproperties\nline2" }, @@ -579,7 +582,10 @@ }, "pig-log4j": { "tag": "version1" - } + }, + "pig-properties": { + "tag": "version1" + } }, "commandId": "7-1", "clusterHostInfo": { http://git-wip-us.apache.org/repos/asf/ambari/blob/b7dd3144/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json index ffa1c55..82b9855 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json @@ -636,6 +636,9 @@ "pig-log4j": { "content": "log4jproperties\nline2" }, + "pig-properties": { + "pig-content": "pigproperties\nline2" + }, "oozie-log4j": { "content": "log4jproperties\nline2" }