AMBARI-14579: Expose pxf-public.classpath and pxf-profiles.xml configs for PXF service (adenissov via jaoki)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5c68080d Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5c68080d Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5c68080d Branch: refs/heads/branch-dev-patch-upgrade Commit: 5c68080da2f96b087f37958d654ae19509f02292 Parents: 377901d Author: Jun Aoki <ja...@apache.org> Authored: Tue Jan 12 12:39:53 2016 -0800 Committer: Nate Cole <nc...@hortonworks.com> Committed: Thu Jan 14 11:43:25 2016 -0500 ---------------------------------------------------------------------- .../PXF/3.0.0/configuration/pxf-profiles.xml | 167 +++++++++++++++++++ .../configuration/pxf-public-classpath.xml | 63 +++++++ .../common-services/PXF/3.0.0/metainfo.xml | 2 + .../PXF/3.0.0/package/scripts/pxf.py | 6 + .../src/test/python/stacks/2.3/PXF/test_pxf.py | 6 + .../python/stacks/2.3/configs/pxf_default.json | 6 + 6 files changed, 250 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/5c68080d/ambari-server/src/main/resources/common-services/PXF/3.0.0/configuration/pxf-profiles.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PXF/3.0.0/configuration/pxf-profiles.xml b/ambari-server/src/main/resources/common-services/PXF/3.0.0/configuration/pxf-profiles.xml new file mode 100644 index 0000000..8319741 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/PXF/3.0.0/configuration/pxf-profiles.xml @@ -0,0 +1,167 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration supports_final="false"> + + <property> + <name>content</name> + <description>PXF profiles definition file</description> + <value><![CDATA[ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +<!-- + PXF profiles definition file. + New profiles can be added in the form: + <profile> + <name>...</name> + <description>...</description> + <plugins> + <plugin_A>...</plugin_A> + <plugin_B>...</plugin_B> + ... + </plugins> + </profile> +--> +<profiles> + <profile> + <name>HBase</name> + <description>This profile is suitable for using when connecting to an HBase data store engine</description> + <plugins> + <fragmenter>org.apache.hawq.pxf.plugins.hbase.HBaseDataFragmenter</fragmenter> + <accessor>org.apache.hawq.pxf.plugins.hbase.HBaseAccessor</accessor> + <resolver>org.apache.hawq.pxf.plugins.hbase.HBaseResolver</resolver> + </plugins> + </profile> + <profile> + <name>Hive</name> + <description>This profile is suitable for using when connecting to Hive</description> + <plugins> + <fragmenter>org.apache.hawq.pxf.plugins.hive.HiveDataFragmenter</fragmenter> + <accessor>org.apache.hawq.pxf.plugins.hive.HiveAccessor</accessor> + <resolver>org.apache.hawq.pxf.plugins.hive.HiveResolver</resolver> + </plugins> + </profile> + <profile> + <name>HiveRC</name> + <description>This profile is suitable only for Hive tables stored in RC files + and serialized with either the ColumnarSerDe or the LazyBinaryColumnarSerDe. + It is much faster than the general purpose Hive profile. + DELIMITER parameter is mandatory. + </description> + <plugins> + <fragmenter>org.apache.hawq.pxf.plugins.hive.HiveInputFormatFragmenter</fragmenter> + <accessor>org.apache.hawq.pxf.plugins.hive.HiveRCFileAccessor</accessor> + <resolver>org.apache.hawq.pxf.plugins.hive.HiveColumnarSerdeResolver</resolver> + </plugins> + </profile> + <profile> + <name>HiveText</name> + <description>This profile is suitable only for Hive tables stored as Text files. + It is much faster than the general purpose Hive profile. + DELIMITER parameter is mandatory. + </description> + <plugins> + <fragmenter>org.apache.hawq.pxf.plugins.hive.HiveInputFormatFragmenter</fragmenter> + <accessor>org.apache.hawq.pxf.plugins.hive.HiveLineBreakAccessor</accessor> + <resolver>org.apache.hawq.pxf.plugins.hive.HiveStringPassResolver</resolver> + </plugins> + </profile> + <profile> + <name>HdfsTextSimple</name> + <description>This profile is suitable for using when reading delimited single line records from plain text files + on HDFS + </description> + <plugins> + <fragmenter>org.apache.hawq.pxf.plugins.hdfs.HdfsDataFragmenter</fragmenter> + <accessor>org.apache.hawq.pxf.plugins.hdfs.LineBreakAccessor</accessor> + <resolver>org.apache.hawq.pxf.plugins.hdfs.StringPassResolver</resolver> + </plugins> + </profile> + <profile> + <name>HdfsTextMulti</name> + <description>This profile is suitable for using when reading delimited single or multi line records (with quoted + linefeeds) from plain text files on HDFS. It is not splittable (non parallel) and slower than HdfsTextSimple. + </description> + <plugins> + <fragmenter>org.apache.hawq.pxf.plugins.hdfs.HdfsDataFragmenter</fragmenter> + <accessor>org.apache.hawq.pxf.plugins.hdfs.QuotedLineBreakAccessor</accessor> + <resolver>org.apache.hawq.pxf.plugins.hdfs.StringPassResolver</resolver> + </plugins> + </profile> + <profile> + <name>Avro</name> + <description>This profile is suitable for using when reading Avro files (i.e fileName.avro)</description> + <plugins> + <fragmenter>org.apache.hawq.pxf.plugins.hdfs.HdfsDataFragmenter</fragmenter> + <accessor>org.apache.hawq.pxf.plugins.hdfs.AvroFileAccessor</accessor> + <resolver>org.apache.hawq.pxf.plugins.hdfs.AvroResolver</resolver> + </plugins> + </profile> + <profile> + <name>SequenceWritable</name> + <description> + Profile for accessing Sequence files serialized with a custom Writable class + usage: pxf://nn:50070/path/to/file?profile=SequenceWritable&data-schema=CustomClass + </description> + <plugins> + <fragmenter>org.apache.hawq.pxf.plugins.hdfs.HdfsDataFragmenter</fragmenter> + <accessor>org.apache.hawq.pxf.plugins.hdfs.SequenceFileAccessor</accessor> + <resolver>org.apache.hawq.pxf.plugins.hdfs.WritableResolver</resolver> + </plugins> + </profile> + <profile> + <name>GemFireXD</name> + <description>This profile is suitable for using when connecting to GemFireXD</description> + <plugins> + <fragmenter>org.apache.hawq.pxf.plugins.gemfirexd.GemFireXDFragmenter</fragmenter> + <accessor>org.apache.hawq.pxf.plugins.gemfirexd.GemFireXDAccessor</accessor> + <resolver>org.apache.hawq.pxf.plugins.gemfirexd.GemFireXDResolver</resolver> + </plugins> + </profile> +</profiles> + ]]> + </value> + <value-attributes> + <show-property-name>false</show-property-name> + </value-attributes> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/5c68080d/ambari-server/src/main/resources/common-services/PXF/3.0.0/configuration/pxf-public-classpath.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PXF/3.0.0/configuration/pxf-public-classpath.xml b/ambari-server/src/main/resources/common-services/PXF/3.0.0/configuration/pxf-public-classpath.xml new file mode 100644 index 0000000..7538ee5 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/PXF/3.0.0/configuration/pxf-public-classpath.xml @@ -0,0 +1,63 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration supports_final="false"> + + <property> + <name>content</name> + <description>PXF service's classpath file</description> + <value> +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +############################################################## +# This is PXF service's classpath file +# Any connector should be added here, one resource per line. +# e.g. /usr/lib/myconnector/myconnector.jar +# Wildcard characters (*,[],?) are supported for the file names +# e.g. /usr/lib/myconnector/*.jar +# /usr/lib/myconnector/myconnector*[0-9].jar +############################################################## + + </value> + <value-attributes> + <show-property-name>false</show-property-name> + </value-attributes> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/5c68080d/ambari-server/src/main/resources/common-services/PXF/3.0.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PXF/3.0.0/metainfo.xml b/ambari-server/src/main/resources/common-services/PXF/3.0.0/metainfo.xml index 0548ab5..cb2411c 100644 --- a/ambari-server/src/main/resources/common-services/PXF/3.0.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/PXF/3.0.0/metainfo.xml @@ -67,6 +67,8 @@ </osSpecifics> <configuration-dependencies> + <config-type>pxf-profiles</config-type> + <config-type>pxf-public-classpath</config-type> <config-type>pxf-site</config-type> </configuration-dependencies> http://git-wip-us.apache.org/repos/asf/ambari/blob/5c68080d/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/pxf.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/pxf.py b/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/pxf.py index caae9bf..59a8b43 100644 --- a/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/pxf.py +++ b/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/pxf.py @@ -100,6 +100,12 @@ class Pxf(Script): shutil.copy2("{0}/pxf-privatehdp.classpath".format(params.pxf_conf_dir), "{0}/pxf-private.classpath".format(params.pxf_conf_dir)) + File('{0}/pxf-public.classpath'.format(params.pxf_conf_dir), + content = params.config['configurations']['pxf-public-classpath']['content'].lstrip()) + + File('{0}/pxf-profiles.xml'.format(params.pxf_conf_dir), + content = params.config['configurations']['pxf-profiles']['content'].lstrip()) + if params.security_enabled: pxf_site_dict = dict(params.config['configurations']['pxf-site']) pxf_site_dict['pxf.service.kerberos.principal'] = "{0}/_HOST@{1}".format(params.pxf_user, params.realm_name) http://git-wip-us.apache.org/repos/asf/ambari/blob/5c68080d/ambari-server/src/test/python/stacks/2.3/PXF/test_pxf.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/PXF/test_pxf.py b/ambari-server/src/test/python/stacks/2.3/PXF/test_pxf.py index 3d803c5..aaee2a5 100644 --- a/ambari-server/src/test/python/stacks/2.3/PXF/test_pxf.py +++ b/ambari-server/src/test/python/stacks/2.3/PXF/test_pxf.py @@ -42,6 +42,12 @@ class TestPxf(RMFTestCase): self.assertResourceCalled('File', '/etc/pxf/conf/pxf-env.sh', content=Template('pxf-env.j2')) + self.assertResourceCalled('File', '/etc/pxf/conf/pxf-public.classpath', + content = self.getConfig()['configurations']['pxf-public-classpath']['content'].lstrip()) + + self.assertResourceCalled('File', '/etc/pxf/conf/pxf-profiles.xml', + content = self.getConfig()['configurations']['pxf-profiles']['content'].lstrip()) + self.assertResourceCalled('XmlConfig', 'pxf-site.xml', conf_dir='/etc/pxf/conf', configurations=self.getConfig()['configurations']['pxf-site'], http://git-wip-us.apache.org/repos/asf/ambari/blob/5c68080d/ambari-server/src/test/python/stacks/2.3/configs/pxf_default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/configs/pxf_default.json b/ambari-server/src/test/python/stacks/2.3/configs/pxf_default.json index 310d9ef..a042dfd 100644 --- a/ambari-server/src/test/python/stacks/2.3/configs/pxf_default.json +++ b/ambari-server/src/test/python/stacks/2.3/configs/pxf_default.json @@ -43,6 +43,12 @@ "pxf-site": { "a": "b" }, + "pxf-public-classpath": { + "content": " classpath" + }, + "pxf-profiles": { + "content": " profile" + }, "cluster-env": { "security_enabled": "false", "ignore_groupsusers_create": "false",
