Repository: ambari Updated Branches: refs/heads/trunk 8d462419e -> 566db4afd
AMBARI-19833. HDP 3.0 TP - support for Tez with configs, kerberos, widgets, metrics, quicklinks, and themes.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/566db4af Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/566db4af Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/566db4af Branch: refs/heads/trunk Commit: 566db4afd1e06538cf6eb427f45a4c0c6661468e Parents: 8d46241 Author: Vitaly Brodetskyi <vbrodets...@hortonworks.com> Authored: Fri Mar 17 21:13:21 2017 +0200 Committer: Vitaly Brodetskyi <vbrodets...@hortonworks.com> Committed: Fri Mar 17 21:13:21 2017 +0200 ---------------------------------------------------------------------- .../TEZ/0.9.0.3.0/configuration/tez-env.xml | 87 +++ .../TEZ/0.9.0.3.0/configuration/tez-site.xml | 523 +++++++++++++++++++ .../common-services/TEZ/0.9.0.3.0/kerberos.json | 24 + .../common-services/TEZ/0.9.0.3.0/metainfo.xml | 111 ++++ .../TEZ/0.9.0.3.0/package/scripts/params.py | 29 + .../0.9.0.3.0/package/scripts/params_linux.py | 116 ++++ .../0.9.0.3.0/package/scripts/params_windows.py | 54 ++ .../0.9.0.3.0/package/scripts/pre_upgrade.py | 59 +++ .../0.9.0.3.0/package/scripts/service_check.py | 111 ++++ .../TEZ/0.9.0.3.0/package/scripts/tez.py | 76 +++ .../TEZ/0.9.0.3.0/package/scripts/tez_client.py | 137 +++++ .../stacks/HDP/3.0/services/TEZ/metainfo.xml | 27 + 12 files changed, 1354 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/566db4af/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/configuration/tez-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/configuration/tez-env.xml b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/configuration/tez-env.xml new file mode 100644 index 0000000..649aa68 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/configuration/tez-env.xml @@ -0,0 +1,87 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_adding_forbidden="true"> + <property> + <name>tez_user</name> + <display-name>Tez User</display-name> + <value>tez</value> + <property-type>USER</property-type> + <description/> + <value-attributes> + <type>user</type> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <!-- tez-env.sh --> + <property> + <name>content</name> + <display-name>tez-env template</display-name> + <description>This is the jinja template for tez-env.sh file</description> + <value> +# Tez specific configuration +export TEZ_CONF_DIR={{config_dir}} + +# Set HADOOP_HOME to point to a specific hadoop install directory +export HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}} + +# The java implementation to use. +export JAVA_HOME={{java64_home}} + </value> + <value-attributes> + <type>content</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>enable_heap_dump</name> + <value>false</value> + <description>Enable or disable taking Heap Dump. (true/false)</description> + <display-name>Enable heap dump</display-name> + <value-attributes> + <type>value-list</type> + <overridable>false</overridable> + <entries> + <entry> + <value>true</value> + <label>Enabled</label> + </entry> + <entry> + <value>false</value> + <label>Disabled</label> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>heap_dump_location</name> + <value>/tmp</value> + <description>Location for heap dump file</description> + <display-name>Heap dump location</display-name> + <on-ambari-upgrade add="false"/> + <value-attributes> + <empty-value-valid>false</empty-value-valid> + </value-attributes> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/566db4af/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/configuration/tez-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/configuration/tez-site.xml b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/configuration/tez-site.xml new file mode 100644 index 0000000..5c17044 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/configuration/tez-site.xml @@ -0,0 +1,523 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration supports_final="true"> + <property> + <name>tez.lib.uris</name> + <value>/hdp/apps/${hdp.version}/tez/tez.tar.gz</value> + <description>Comma-delimited list of the location of the Tez libraries which will be localized for DAGs. + Specifying a single .tar.gz or .tgz assumes that a compressed version of the tez libs is being used. This is uncompressed into a tezlibs directory when running containers, and tezlibs/;tezlibs/lib/ are added to the classpath (after . and .*). + If multiple files are specified - files are localized as regular files, contents of directories are localized as regular files (non-recursive). + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.cluster.additional.classpath.prefix</name> + <value>/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure</value> + <description/> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.am.log.level</name> + <value>INFO</value> + <description>Root Logging level passed to the Tez app master</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.generate.debug.artifacts</name> + <value>false</value> + <description>Generate debug artifacts such as a text representation of the submitted DAG plan</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.staging-dir</name> + <value>/tmp/${user.name}/staging</value> + <description>The staging dir used while submitting DAGs</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.am.resource.memory.mb</name> + <value>1536</value> + <description>The amount of memory to be used by the AppMaster. + Used only if the value is not specified explicitly by the DAG definition. + </description> + <value-attributes> + <type>int</type> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.am.launch.cmd-opts</name> + <value>-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC</value> + <description>Java options for the Tez AppMaster process. The Xmx value is derived based on tez.am.resource.memory.mb and is 80% of the value by default. + Used only if the value is not specified explicitly by the DAG definition. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.am.launch.cluster-default.cmd-opts</name> + <value>-server -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value> + <description>Cluster default Java options for the Tez AppMaster process. These will be prepended to the properties specified via tez.am.launch.cmd-opts</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.am.launch.env</name> + <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value> + <description> + Additional execution environment entries for tez. This is not an additive property. You must preserve the original value if + you want to have access to native libraries. + Used only if the value is not specified explicitly by the DAG definition. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.task.resource.memory.mb</name> + <value>1536</value> + <description>The amount of memory to be used by launched tasks. + Used only if the value is not specified explicitly by the DAG definition. + </description> + <depends-on> + <property> + <type>yarn-site</type> + <name>yarn.scheduler.minimum-allocation-mb</name> + </property> + <property> + <type>yarn-site</type> + <name>yarn.scheduler.maximum-allocation-mb</name> + </property> + </depends-on> + <value-attributes> + <type>int</type> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.task.launch.cmd-opts</name> + <value>-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC{{heap_dump_opts}}</value> + <description>Java options for tasks. The Xmx value is derived based on tez.task.resource.memory.mb and is 80% of this value by default. + Used only if the value is not specified explicitly by the DAG definition. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.task.launch.cluster-default.cmd-opts</name> + <value>-server -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value> + <description>Cluster default Java options for tasks. These will be prepended to the properties specified via tez.task.launch.cmd-opts</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.task.launch.env</name> + <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value> + <description> + Additional execution environment entries for tez. This is not an additive property. You must preserve the original value if + you want to have access to native libraries. + Used only if the value is not specified explicitly by the DAG definition. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.shuffle-vertex-manager.min-src-fraction</name> + <value>0.2</value> + <description>In case of a ScatterGather connection, the fraction of source tasks which should + complete before tasks for the current vertex are schedule + </description> + <value-attributes> + <type>float</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.shuffle-vertex-manager.max-src-fraction</name> + <value>0.4</value> + <description>In case of a ScatterGather connection, once this fraction of source tasks have + completed, all tasks on the current vertex can be scheduled. Number of tasks ready for + scheduling on the current vertex scales linearly between min-fraction and max-fraction + </description> + <value-attributes> + <type>float</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.am.am-rm.heartbeat.interval-ms.max</name> + <value>250</value> + <description>The maximum heartbeat interval between the AM and RM in milliseconds</description> + <value-attributes> + <type>int</type> + <unit>ms</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.grouping.split-waves</name> + <value>1.7</value> + <description>The multiplier for available queue capacity when determining number of tasks for + a Vertex. 1.7 with 100% queue available implies generating a number of tasks roughly equal + to 170% of the available containers on the queue + </description> + <value-attributes> + <type>float</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.grouping.min-size</name> + <value>16777216</value> + <description>Lower bound on the size (in bytes) of a grouped split, to avoid generating + too many splits + </description> + <value-attributes> + <type>int</type> + <unit>Bytes</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.grouping.max-size</name> + <value>1073741824</value> + <description>Upper bound on the size (in bytes) of a grouped split, to avoid generating + excessively large split + </description> + <value-attributes> + <type>int</type> + <unit>Bytes</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.am.container.reuse.enabled</name> + <value>true</value> + <description>Configuration to specify whether container should be reused</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.am.container.reuse.rack-fallback.enabled</name> + <value>true</value> + <description>Whether to reuse containers for rack local tasks. Active only if reuse is enabled + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.am.container.reuse.non-local-fallback.enabled</name> + <value>false</value> + <description>Whether to reuse containers for non-local tasks. Active only if reuse is enabled + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.am.container.idle.release-timeout-min.millis</name> + <value>10000</value> + <description>The minimum amount of time to hold on to a container that is idle. Only active when reuse is enabled.</description> + <value-attributes> + <type>int</type> + <unit>ms</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.am.container.idle.release-timeout-max.millis</name> + <value>20000</value> + <description>The maximum amount of time to hold on to a container if no task can be assigned to it immediately. Only active when reuse is enabled.</description> + <value-attributes> + <type>int</type> + <unit>ms</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.am.container.reuse.locality.delay-allocation-millis</name> + <value>250</value> + <description>The amount of time to wait before assigning a container to the next level of + locality. NODE -> RACK -> NON_LOCAL + </description> + <value-attributes> + <type>int</type> + <unit>ms</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.am.max.app.attempts</name> + <value>2</value> + <description>Specifies the total number of time the app master will run in case recovery is triggered</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.am.maxtaskfailures.per.node</name> + <value>10</value> + <description>The maximum number of allowed task attempt failures on a node before + it gets marked as blacklisted + </description> + <value-attributes> + <type>int</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.task.am.heartbeat.counter.interval-ms.max</name> + <value>4000</value> + <description>Time interval at which task counters are sent to the AM</description> + <value-attributes> + <type>int</type> + <unit>ms</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.task.get-task.sleep.interval-ms.max</name> + <value>200</value> + <description>The maximum amount of time, in seconds, to wait before a task asks an AM for + another task + </description> + <value-attributes> + <type>int</type> + <unit>ms</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.task.max-events-per-heartbeat</name> + <value>500</value> + <description>Maximum number of of events to fetch from the AM by the tasks in a single heartbeat.</description> + <value-attributes> + <type>int</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.session.client.timeout.secs</name> + <value>-1</value> + <description>Time (in seconds) to wait for AM to come up when trying to submit a DAG from + the client + </description> + <value-attributes> + <type>int</type> + <unit>seconds</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.session.am.dag.submit.timeout.secs</name> + <value>300</value> + <description>Time (in seconds) for which the Tez AM should wait for a DAG to be submitted + before shutting down + </description> + <value-attributes> + <type>int</type> + <unit>seconds</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.runtime.compress</name> + <value>true</value> + <description>Whether intermediate data should be compressed or not</description> + <value-attributes> + <type>boolean</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.runtime.compress.codec</name> + <value>org.apache.hadoop.io.compress.SnappyCodec</value> + <description>The coded to be used if compressing intermediate data. Only + applicable if tez.runtime.compress is enabled + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.runtime.unordered.output.buffer.size-mb</name> + <value>100</value> + <description>The size of the buffer when output does not require to be sorted</description> + <depends-on> + <property> + <type>tez-site</type> + <name>tez.task.resource.memory.mb</name> + </property> + </depends-on> + <value-attributes> + <type>int</type> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.runtime.convert.user-payload.to.history-text</name> + <value>false</value> + <description>Whether to publish configuration information to History logger</description> + <value-attributes> + <type>boolean</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.use.cluster.hadoop-libs</name> + <value>false</value> + <description>This being true implies that the deployment is relying on hadoop jars being available on the cluster on all nodes.</description> + <value-attributes> + <type>boolean</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.am.tez-ui.history-url.template</name> + <value>__HISTORY_URL_BASE__?viewPath=%2F%23%2Ftez-app%2F__APPLICATION_ID__</value> + <description>Template to generate the History URL for a particular Tez Application. + Template replaces __APPLICATION_ID__ with the actual applicationId and + __HISTORY_URL_BASE__ with the value from the tez.tez-ui.history-url.base config property + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.tez-ui.history-url.base</name> + <description/> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.am.view-acls</name> + <value>*</value> + <description>AM view ACLs. This allows the specified users/groups to view the status of the AM and all DAGs that run within this AM. + Value format: Comma separated list of users, followed by whitespace, followed by a comma separated list of groups. + </description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.runtime.optimize.local.fetch</name> + <value>true</value> + <description>If the shuffle input is on the local host bypass the http fetch and access the files directly.</description> + <value-attributes> + <type>boolean</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.task.generate.counters.per.io</name> + <value>true</value> + <description>Whether to generate counters on a per-edge basis for a Tez DAG. Helpful for in-depth analysis.</description> + <value-attributes> + <type>boolean</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.runtime.sorter.class</name> + <value>PIPELINED</value> + <description>Which sorter implementation to use</description> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>PIPELINED</value> + <label>Pipelined Sorter</label> + </entry> + <entry> + <value>LEGACY</value> + <label>Legacy Sorter</label> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.runtime.pipelined.sorter.sort.threads</name> + <value>2</value> + <description>Tez runtime pipelined sorter sort threads</description> + <value-attributes> + <type>int</type> + </value-attributes> + <depends-on> + <property> + <type>tez-site</type> + <name>tez.runtime.sorter.class</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.runtime.io.sort.mb</name> + <value>272</value> + <description>The size of the sort buffer when output needs to be sorted</description> + <value-attributes> + <type>int</type> + <unit>MB</unit> + </value-attributes> + <depends-on> + <property> + <type>tez-site</type> + <name>tez.runtime.sorter.class</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.history.logging.service.class</name> + <value>org.apache.tez.dag.history.logging.ats.ATSV15HistoryLoggingService</value> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.counters.max</name> + <value>10000</value> + <description>The number of allowed counters for the executing DAG</description> + <value-attributes> + <type>int</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.counters.max.groups</name> + <value>3000</value> + <description>The number of allowed counter groups for the executing DAG</description> + <value-attributes> + <type>int</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.runtime.shuffle.fetch.buffer.percent</name> + <value>0.6</value> + <description>Fraction (0-1) of the available memory which can be used to + retain shuffled data</description> + <value-attributes> + <type>float</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.runtime.shuffle.memory.limit.percent</name> + <value>0.25</value> + <description>This property determines the maximum size of a shuffle segment + which can be fetched to memory. Fraction (0-1) of shuffle memory + (after applying tez.runtime.shuffle.fetch.buffer.percent)</description> + <value-attributes> + <type>float</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>tez.history.logging.timeline-cache-plugin.old-num-dags-per-group</name> + <value>5</value> + <description>DAGs per group</description> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/566db4af/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/kerberos.json b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/kerberos.json new file mode 100644 index 0000000..00c8ac5 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/kerberos.json @@ -0,0 +1,24 @@ +{ + "services": [ + { + "name": "TEZ", + "components": [ + { + "name": "TEZ_CLIENT", + "identities": [ + { + "name": "/HDFS/NAMENODE/hdfs" + } + ], + "configurations": [ + { + "tez-site": { + "tez.am.view-acls": "" + } + } + ] + } + ] + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/566db4af/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/metainfo.xml b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/metainfo.xml new file mode 100644 index 0000000..8ca01f3 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/metainfo.xml @@ -0,0 +1,111 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>TEZ</name> + <displayName>Tez</displayName> + <comment>Tez is the next generation Hadoop Query Processing framework written on top of YARN.</comment> + <version>0.9.0.3.0</version> + <components> + <component> + <name>TEZ_CLIENT</name> + <displayName>Tez Client</displayName> + <cardinality>1+</cardinality> + <versionAdvertised>true</versionAdvertised> + <category>CLIENT</category> + <commandScript> + <script>scripts/tez_client.py</script> + <scriptType>PYTHON</scriptType> + <timeout>1200</timeout> + </commandScript> + <configFiles> + <configFile> + <type>xml</type> + <fileName>tez-site.xml</fileName> + <dictionaryName>tez-site</dictionaryName> + </configFile> + <configFile> + <type>env</type> + <fileName>tez-env.sh</fileName> + <dictionaryName>tez-env</dictionaryName> + </configFile> + </configFiles> + <dependencies> + <dependency> + <name>HDFS/HDFS_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>YARN/YARN_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> + </component> + </components> + <osSpecifics> + <osSpecific> + <osFamily>redhat7,amazon2015,redhat6,suse11,suse12</osFamily> + <packages> + <package> + <name>tez_${stack_version}</name> + </package> + </packages> + </osSpecific> + <osSpecific> + <osFamily>debian7,ubuntu12,ubuntu14,ubuntu16</osFamily> + <packages> + <package> + <name>tez-${stack_version}</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <commandScript> + <script>scripts/service_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + + <requiredServices> + <service>YARN</service> + </requiredServices> + + <configuration-dependencies> + <config-type>tez-site</config-type> + <config-type>tez-env</config-type> + </configuration-dependencies> + + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/566db4af/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/params.py new file mode 100644 index 0000000..c01dc62 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/params.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from ambari_commons import OSCheck +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions.copy_tarball import get_sysprep_skip_copy_tarballs_hdfs + +if OSCheck.is_windows_family(): + from params_windows import * +else: + from params_linux import * + +sysprep_skip_copy_tarballs_hdfs = get_sysprep_skip_copy_tarballs_hdfs() \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/566db4af/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/params_linux.py new file mode 100644 index 0000000..5a028bd --- /dev/null +++ b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/params_linux.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +import os + +from resource_management.libraries.resources import HdfsResource +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.version import format_stack_version +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions import get_kinit_path +from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions.get_architecture import get_architecture + +# server configurations +config = Script.get_config() +tmp_dir = Script.get_tmp_dir() + +architecture = get_architecture() + +stack_name = default("/hostLevelParams/stack_name", None) +stack_root = Script.get_stack_root() + +# This is expected to be of the form #.#.#.# +stack_version_unformatted = config['hostLevelParams']['stack_version'] +stack_version_formatted = format_stack_version(stack_version_unformatted) + +# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade +version = default("/commandParams/version", None) + +# default hadoop parameters +hadoop_home = '/usr' +hadoop_bin_dir = stack_select.get_hadoop_dir("bin") +hadoop_conf_dir = conf_select.get_hadoop_conf_dir() +tez_etc_dir = "/etc/tez" +config_dir = "/etc/tez/conf" +tez_examples_jar = "/usr/lib/tez/tez-mapreduce-examples*.jar" + +# hadoop parameters for stacks that support rolling_upgrade +if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted): + tez_examples_jar = "{stack_root}/current/tez-client/tez-examples*.jar" + +# tez only started linking <stack-root>/x.x.x.x/tez-client/conf in config_versioning +if stack_version_formatted and check_stack_feature(StackFeature.CONFIG_VERSIONING, stack_version_formatted): + # !!! use realpath for now since the symlink exists but is broken and a + # broken symlink messes with the DirectoryProvider class + config_path = os.path.join(stack_root, "current/tez-client/conf") + config_dir = os.path.realpath(config_path) + +# Heap dump related +heap_dump_enabled = default('/configurations/tez-env/enable_heap_dump', None) +heap_dump_opts = "" # Empty if 'heap_dump_enabled' is False. +if heap_dump_enabled: + heap_dump_path = default('/configurations/tez-env/heap_dump_location', "/tmp") + heap_dump_opts = " -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath="+heap_dump_path + +kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) +security_enabled = config['configurations']['cluster-env']['security_enabled'] +smokeuser = config['configurations']['cluster-env']['smokeuser'] +smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] +hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] +hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] + +java64_home = config['hostLevelParams']['java_home'] + +tez_user = config['configurations']['tez-env']['tez_user'] +user_group = config['configurations']['cluster-env']['user_group'] +tez_env_sh_template = config['configurations']['tez-env']['content'] + +hdfs_site = config['configurations']['hdfs-site'] +default_fs = config['configurations']['core-site']['fs.defaultFS'] + +dfs_type = default("/commandParams/dfs_type", "") + +import functools +#create partial functions with common arguments for every HdfsResource call +#to create/delete/copyfromlocal hdfs directories/files we need to call params.HdfsResource in code +HdfsResource = functools.partial( + HdfsResource, + user=hdfs_user, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", + security_enabled = security_enabled, + keytab = hdfs_user_keytab, + kinit_path_local = kinit_path_local, + hadoop_bin_dir = hadoop_bin_dir, + hadoop_conf_dir = hadoop_conf_dir, + principal_name = hdfs_principal_name, + hdfs_site = hdfs_site, + default_fs = default_fs, + immutable_paths = get_not_managed_resources(), + dfs_type = dfs_type +) + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/566db4af/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/params_windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/params_windows.py new file mode 100644 index 0000000..64c2a53 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/params_windows.py @@ -0,0 +1,54 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import os + +from resource_management.libraries.functions.get_stack_version import get_stack_version +from resource_management.libraries.script.script import Script + +config = Script.get_config() +hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"] +user_group = config["configurations"]["cluster-env"]["user_group"] +tez_user = hadoop_user +tez_home_dir = None +tez_conf_dir = "conf" + +try: + hadoop_classpath_prefix_template = config["configurations"]["tez-site"]["tez.cluster.additional.classpath.prefix"] +except KeyError: + hadoop_classpath_prefix_template = "" + +stack_version_formatted = "" + +stack_root = None +try: + stack_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"], "..")) +except: + pass + +def refresh_tez_state_dependent_params(): + global tez_home_dir, tez_conf_dir, stack_version_formatted + tez_home_dir = os.environ["TEZ_HOME"] + tez_conf_dir = os.path.join(tez_home_dir, "conf") + # this is not available on INSTALL action because <stack-selector-tool> is not available + stack_version_formatted = get_stack_version("tez") + + +if os.environ.has_key("TEZ_HOME"): + refresh_tez_state_dependent_params() http://git-wip-us.apache.org/repos/asf/ambari/blob/566db4af/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/pre_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/pre_upgrade.py b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/pre_upgrade.py new file mode 100644 index 0000000..d182efe --- /dev/null +++ b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/pre_upgrade.py @@ -0,0 +1,59 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management.libraries.script import Script +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.copy_tarball import copy_to_hdfs +from resource_management.core.exceptions import Fail + +from resource_management.core.logger import Logger + +class TezPreUpgrade(Script): + + def prepare(self, env): + """ + During the "Upgrade" direction of a Stack Upgrade, it is necessary to ensure that the older tez tarball + has been copied to HDFS. This is an additional check for added robustness. + """ + import params + env.set_params(params) + + Logger.info("Before starting Stack Upgrade, check if tez tarball has been copied to HDFS.") + + if params.stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.stack_version_formatted): + Logger.info("Stack version {0} is sufficient to check if need to copy tez.tar.gz to HDFS.".format(params.stack_version_formatted)) + + # Force it to copy the current version of the tez tarball, rather than the version the RU will go to. + resource_created = copy_to_hdfs( + "tez", + params.user_group, + params.hdfs_user, + use_upgrading_version_during_upgrade=False, + skip=params.sysprep_skip_copy_tarballs_hdfs) + if resource_created: + params.HdfsResource(None, action="execute") + else: + raise Fail("Could not copy tez tarball to HDFS.") + +if __name__ == "__main__": + TezPreUpgrade().execute() + http://git-wip-us.apache.org/repos/asf/ambari/blob/566db4af/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/service_check.py new file mode 100644 index 0000000..c903d35 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/service_check.py @@ -0,0 +1,111 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +import os +from resource_management.libraries.script import Script +from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.resources.execute_hadoop import ExecuteHadoop +from resource_management.libraries.functions import format +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.copy_tarball import copy_to_hdfs +from resource_management.core.resources.system import File, Execute + +from ambari_commons import OSConst +from ambari_commons.os_family_impl import OsFamilyImpl + +from resource_management.core.logger import Logger + +class TezServiceCheck(Script): + pass + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class TezServiceCheckLinux(TezServiceCheck): + def service_check(self, env): + import params + env.set_params(params) + + path_to_tez_jar = format(params.tez_examples_jar) + wordcount_command = format("jar {path_to_tez_jar} orderedwordcount /tmp/tezsmokeinput/sample-tez-test /tmp/tezsmokeoutput/") + test_command = format("fs -test -e /tmp/tezsmokeoutput/_SUCCESS") + + File(format("{tmp_dir}/sample-tez-test"), + content = "foo\nbar\nfoo\nbar\nfoo", + mode = 0755 + ) + + params.HdfsResource("/tmp/tezsmokeoutput", + action = "delete_on_execute", + type = "directory" + ) + + params.HdfsResource("/tmp/tezsmokeinput", + action = "create_on_execute", + type = "directory", + owner = params.smokeuser, + ) + params.HdfsResource("/tmp/tezsmokeinput/sample-tez-test", + action = "create_on_execute", + type = "file", + owner = params.smokeuser, + source = format("{tmp_dir}/sample-tez-test"), + ) + + if params.stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.stack_version_formatted): + copy_to_hdfs("tez", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs) + + params.HdfsResource(None, action = "execute") + + if params.security_enabled: + kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};") + Execute(kinit_cmd, + user=params.smokeuser + ) + + ExecuteHadoop(wordcount_command, + tries = 3, + try_sleep = 5, + user = params.smokeuser, + conf_dir = params.hadoop_conf_dir, + bin_dir = params.hadoop_bin_dir + ) + + ExecuteHadoop(test_command, + tries = 10, + try_sleep = 6, + user = params.smokeuser, + conf_dir = params.hadoop_conf_dir, + bin_dir = params.hadoop_bin_dir + ) + + +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class TezServiceCheckWindows(TezServiceCheck): + def service_check(self, env): + import params + env.set_params(params) + smoke_cmd = os.path.join(params.stack_root,"Run-SmokeTests.cmd") + service = "TEZ" + Execute(format("cmd /C {smoke_cmd} {service}"), logoutput=True, user=params.tez_user) + + +if __name__ == "__main__": + TezServiceCheck().execute() + http://git-wip-us.apache.org/repos/asf/ambari/blob/566db4af/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/tez.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/tez.py b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/tez.py new file mode 100644 index 0000000..dfa6501 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/tez.py @@ -0,0 +1,76 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +# Python Imports +import os + +# Local Imports +from resource_management.core.resources.system import Directory, File +from resource_management.libraries.resources.xml_config import XmlConfig +from resource_management.libraries.functions.format import format +from resource_management.core.source import InlineTemplate +from ambari_commons import OSConst +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl + +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +def tez(config_dir): + """ + Write out tez-site.xml and tez-env.sh to the config directory. + :param config_dir: Which config directory to save configs to, which is different during rolling upgrade. + """ + import params + + Directory(params.tez_etc_dir, mode=0755) + + Directory(config_dir, + owner = params.tez_user, + group = params.user_group, + create_parents = True) + + XmlConfig( "tez-site.xml", + conf_dir = config_dir, + configurations = params.config['configurations']['tez-site'], + configuration_attributes=params.config['configuration_attributes']['tez-site'], + owner = params.tez_user, + group = params.user_group, + mode = 0664) + + tez_env_file_path = os.path.join(config_dir, "tez-env.sh") + File(tez_env_file_path, + owner=params.tez_user, + content=InlineTemplate(params.tez_env_sh_template), + mode=0555) + + +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def tez(config_dir): + """ + Write out tez-site.xml and tez-env.sh to the config directory. + :param config_dir: Directory to write configs to. + """ + import params + XmlConfig("tez-site.xml", + conf_dir=config_dir, + configurations=params.config['configurations']['tez-site'], + owner=params.tez_user, + mode="f", + configuration_attributes=params.config['configuration_attributes']['tez-site'] + ) + http://git-wip-us.apache.org/repos/asf/ambari/blob/566db4af/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/tez_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/tez_client.py b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/tez_client.py new file mode 100644 index 0000000..8018f0f --- /dev/null +++ b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/tez_client.py @@ -0,0 +1,137 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +import os +import urlparse + +from ambari_commons import OSConst +from ambari_commons.inet_utils import download_file +from ambari_commons.os_family_impl import OsFamilyImpl +from ambari_commons.os_utils import copy_file, extract_path_component + +from resource_management.core.exceptions import ClientComponentHasNoStatus +from resource_management.core.source import InlineTemplate +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.get_stack_version import get_stack_version +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions.default import default +from resource_management.core.logger import Logger + +from tez import tez + +class TezClient(Script): + + def configure(self, env, config_dir=None, upgrade_type=None): + """ + Write tez-site.xml and tez-env.sh to the config directory + :param env: Python Environment + :param config_dir: During rolling upgrade, which config directory to save configs to. + E.g., /usr/$STACK/current/tez-client/conf + """ + import params + env.set_params(params) + tez(config_dir) + + def status(self, env): + raise ClientComponentHasNoStatus() + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class TezClientLinux(TezClient): + + def get_component_name(self): + return "hadoop-client" + + def stack_upgrade_save_new_config(self, env): + """ + Because this gets called during a Rolling Upgrade, the new tez configs have already been saved, so we must be + careful to only call configure() on the directory of the new version. + :param env: + """ + import params + env.set_params(params) + + conf_select_name = "tez" + base_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + config_dir = self.get_config_dir_during_stack_upgrade(env, base_dir, conf_select_name) + + if config_dir: + Logger.info("stack_upgrade_save_new_config(): Calling conf-select on %s using version %s" % (conf_select_name, str(params.version))) + + # Because this script was called from ru_execute_tasks.py which already enters an Environment with its own basedir, + # must change it now so this function can find the Jinja Templates for the service. + env.config.basedir = base_dir + conf_select.select(params.stack_name, conf_select_name, params.version) + self.configure(env, config_dir=config_dir) + + def pre_upgrade_restart(self, env, upgrade_type=None): + import params + env.set_params(params) + + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): + conf_select.select(params.stack_name, "tez", params.version) + conf_select.select(params.stack_name, "hadoop", params.version) + stack_select.select("hadoop-client", params.version) + + def install(self, env): + import params + self.install_packages(env) + self.configure(env, config_dir=params.config_dir) + +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class TezClientWindows(TezClient): + def install(self, env): + import params + if params.tez_home_dir is None: + self.install_packages(env) + params.refresh_tez_state_dependent_params() + env.set_params(params) + self._install_lzo_support_if_needed(params) + self.configure(env, config_dir=params.tez_conf_dir) + + def _install_lzo_support_if_needed(self, params): + hadoop_classpath_prefix = self._expand_hadoop_classpath_prefix(params.hadoop_classpath_prefix_template, params.config['configurations']['tez-site']) + + hadoop_lzo_dest_path = extract_path_component(hadoop_classpath_prefix, "hadoop-lzo-") + if hadoop_lzo_dest_path: + hadoop_lzo_file = os.path.split(hadoop_lzo_dest_path)[1] + + config = Script.get_config() + file_url = urlparse.urljoin(config['hostLevelParams']['jdk_location'], hadoop_lzo_file) + hadoop_lzo_dl_path = os.path.join(config["hostLevelParams"]["agentCacheDir"], hadoop_lzo_file) + download_file(file_url, hadoop_lzo_dl_path) + #This is for protection against configuration changes. It will infect every new destination with the lzo jar, + # but since the classpath points to the jar directly we're getting away with it. + if not os.path.exists(hadoop_lzo_dest_path): + copy_file(hadoop_lzo_dl_path, hadoop_lzo_dest_path) + + def _expand_hadoop_classpath_prefix(self, hadoop_classpath_prefix_template, configurations): + import resource_management + + hadoop_classpath_prefix_obj = InlineTemplate(hadoop_classpath_prefix_template, configurations_dict=configurations, + extra_imports=[resource_management, resource_management.core, + resource_management.core.source]) + hadoop_classpath_prefix = hadoop_classpath_prefix_obj.get_content() + return hadoop_classpath_prefix + +if __name__ == "__main__": + TezClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/566db4af/ambari-server/src/main/resources/stacks/HDP/3.0/services/TEZ/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/TEZ/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/TEZ/metainfo.xml new file mode 100644 index 0000000..f7cab79 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/TEZ/metainfo.xml @@ -0,0 +1,27 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>TEZ</name> + <version>0.9.0.3.0</version> + <extends>common-services/TEZ/0.9.0.3.0</extends> + </service> + </services> +</metainfo>
