Repository: ambari Updated Branches: refs/heads/trunk 856790517 -> af6f6e877
http://git-wip-us.apache.org/repos/asf/ambari/blob/af6f6e87/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/configuration/pig-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/configuration/pig-log4j.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/configuration/pig-log4j.xml deleted file mode 100644 index cbdd452..0000000 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/configuration/pig-log4j.xml +++ /dev/null @@ -1,61 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> - -<configuration supports_final="false"> - - <property> - <name>content</name> - <value> -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# -# - -# ***** Set root logger level to DEBUG and its only appender to A. -log4j.logger.org.apache.pig=info, A - -# ***** A is set to be a ConsoleAppender. -log4j.appender.A=org.apache.log4j.ConsoleAppender -# ***** A uses PatternLayout. -log4j.appender.A.layout=org.apache.log4j.PatternLayout -log4j.appender.A.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n - </value> - </property> - -</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/af6f6e87/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/configuration/pig-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/configuration/pig-properties.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/configuration/pig-properties.xml deleted file mode 100644 index 88e2fea..0000000 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/configuration/pig-properties.xml +++ /dev/null @@ -1,262 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> - -<configuration supports_final="false"> - - <property> - <name>pig-content</name> - <value> -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Pig configuration file. All values can be overwritten by command line arguments. - -# Use the "-h properties" command to see description of the properties - -# log4jconf log4j configuration file -# log4jconf=./conf/log4j.properties - -# a file that contains pig script -#file= - -# load jarfile, colon separated -#jar= - -#verbose print all log messages to screen (default to print only INFO and above to screen) -#verbose=true - -#exectype local|mapreduce, mapreduce is default -#exectype=local - -#the default timezone: if it is not set, the default timezone for this host is used. -#the correct timezone format is the UTC offset: e.g., +08:00. -#pig.datetime.default.tz= - -#pig.logfile= - -#Do not spill temp files smaller than this size (bytes) -#pig.spill.size.threshold=5000000 - -#EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes) -#This should help reduce the number of files being spilled. -#pig.spill.gc.activation.size=40000000 - -#the following two parameters are to help estimate the reducer number -#pig.exec.reducers.bytes.per.reducer=1000000000 -#pig.exec.reducers.max=999 - -#Logging properties -#verbose=false -#brief=false -#debug=INFO -#aggregate.warning=true - -#Performance tuning properties -#pig.cachedbag.memusage=0.2 -#pig.skewedjoin.reduce.memusage=0.3 -#pig.exec.nocombiner=false -#opt.multiquery=true - -#Following parameters are for configuring intermediate storage format -#Supported storage types are seqfile and tfile -#Supported codec types: tfile supports gz(gzip) and lzo, seqfile support gz(gzip), lzo, snappy, bzip2 -#pig.tmpfilecompression=false -#pig.tmpfilecompression.storage=seqfile -#pig.tmpfilecompression.codec=gz - -#pig.noSplitCombination=true - -#pig.exec.mapPartAgg=false -#pig.exec.mapPartAgg.minReduction=10 - -#exectype=mapreduce -#pig.additional.jars=<comma seperated list of jars> -#udf.import.list=<comma seperated list of imports> -#stop.on.failure=false - -#Use this option only when your Pig job will otherwise die because of -#using more counters than hadoop configured limit -#pig.disable.counter=true - -# By default, pig will allow 1GB of data to be replicated using -# the distributed cache when doing fragment-replicated join. -# pig.join.replicated.max.bytes=1000000000 - -# Use this option to turn on UDF timers. This will cause two -# counters to be tracked for every UDF and LoadFunc in your script: -# approx_microsecs measures approximate time spent inside a UDF -# approx_invocations reports the approximate number of times the UDF was invoked -# pig.udf.profile=false - -#When enabled, 'describe' prints a multi-line formatted schema -#(similar to an indended json) rather than on a single line. -#pig.pretty.print.schema=true - -#pig.sql.type=hcat -hcat.bin=c:\hdp\hcatalog-@hcat.version@\\bin\\hcat.py - -############################ SchemaTuple ############################ - -# Setting this value will turn on the SchemaTuple feature (PIG-2632) -# This will attempt to use code generation for more efficient within -# the pig code. This can lead to both CPU, serialization, and memory -# benefits (currently, the potential memory benefits are the largest). - -# This parameter will enable the optimization in all available cases -#pig.schematuple=true - -# Certain cases can be turned off by uncommenting the following. These will -# all be off by default, but will all be turned on if pig.schematuple is set -# to true. - -# This will disable SchemaTuples in the case of udfs. Currently, -# the input to UDF's will be SchemaTuples. - -#pig.schematuple.udf=false - -# This is currently not implemented. In the future, LoadFunc's with known -# schema's should output SchemaTuples - -#pig.schematuple.load=false - -# This will use SchemaTuples in replicated joins. The potential memory saving -# here is significant. It will use SchemaTuples when it builds the HashMap of -# the join key to related values. - -#pig.schematuple.fr_join=false - -# In the current implementation of merge join, all of the Tuples in the left relation -# that share a given key will be stored in a List in memory. This will use SchemaTuples -# instead in that List. - -#pig.schematuple.merge_join=false - -##################################################################### - -##### Set up optional Pig Progress Notification Listener ############ - -# Note that only one PPNL can be set up. If you need several, write a PPNL that will chain them. -# pig.notification.listener = <fully qualified class name of a PPNL implementation> - -# Optionally, you can supply a single String argument to pass to your PPNL. -# pig.notification.listener.arg = <somevalue> - -##################################################################### - -########## Override the default Reducer Estimator logic ############# - -# By default, the logic to estimate the number of reducers to use for a given job lives in: -# org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.InputSizeReducerEstimator -# This logic can be replaced by implementing the following interface: -# org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigReducerEstimator - -# This class will be invoked to estimate the number of reducers to use. -# pig.exec.reducer.estimator = <fully qualified class name of a PigReducerEstimator implementation> - -# Optionally, you can supply a single String argument to pass to your PigReducerEstimator. -# pig.exec.reducer.estimator.arg = <somevalue> - -##################################################################### - -###### Override the default Pig Stats Output Size Reader logic ###### - -# By default, the size of reducers output is computed as the total size of -# output files. But since not every storage is file-based, this logic is not -# always applicable. If that is the case, the logic can be replaced by -# implementing the following interface: -# org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigStatsOutputSizeReader - -# This class will be invoked to compute the size of reducers output. -# pig.stats.output.size.reader = <fully qualified class name of a PigStatsOutputSizeReader implementation> - -# If you need to register more than one reader, you can register them as a comma -# separated list. Every reader implements a boolean supports(POStore sto) method. -# When there are more than one reader, they are consulted in order, and the -# first one whose supports() method returns true will be used. -# -##################################################################### - -#pig.load.default.statements= - -##################################################################### - -########### Override hadoop configs programatically ################# - -# By default, Pig expects hadoop configs (hadoop-site.xml and core-site.xml) -# to be present on the classpath. There are cases when these configs are -# needed to be passed programatically, such as while using the PigServer API. -# In such cases, you can override hadoop configs by setting the property -# "pig.use.overriden.hadoop.configs". -# -# When this property is set to true, Pig ignores looking for hadoop configs -# in the classpath and instead picks it up from Properties/Configuration -# object passed to it. - -# pig.use.overriden.hadoop.configs=false -# -###################################################################### - -# Check if the script needs to check multiple stores writing -# to the same location. When set to true, stops the execution -# of script right away. -pig.location.check.strict=false - -###################################################################### - -# This key is used to define the default load func. Pig will fallback -# on PigStorage as default in case this is undefined. - -# pig.default.load.func=<fully qualified class name of a LoadFunc implementation> -# For eg, pig.default.load.func=org.apache.pig.custom.MyCustomStorage - -# This key is used to define the default store func. Pig will fallback -# on PigStorage as default in case this is undefined. - -# pig.default.store.func=<fully qualified class name of a StoreFunc implementation> -# For eg, pig.default.store.func=org.apache.pig.custom.MyCustomStorage - -# This option is used to define whether to support recovery to handle the -# application master getting restarted. -# pig.output.committer.recovery.support=true - -# Set this option to true if you need to use the old partition filter optimizer. -# Note: Old filter optimizer PColFilterOptimizer will be deprecated in the future. -# pig.exec.useOldPartitionFilterOptimizer=true - - </value> - </property> - -</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/af6f6e87/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/metainfo.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/metainfo.xml index 68f7566..03de475 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/metainfo.xml @@ -20,56 +20,21 @@ <services> <service> <name>PIG</name> - <displayName>Pig</displayName> - <comment>Scripting platform for analyzing large datasets</comment> - <version>0.12.0.2.0</version> + <extends>common-services/PIG/0.12.0.2.0</extends> + <version>0.12.1.2.1.1.0</version> <components> <component> <name>PIG</name> - <displayName>Pig</displayName> - <category>CLIENT</category> - <cardinality>0+</cardinality> - <commandScript> - <script>scripts/pig_client.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> <configFiles> <configFile> <type>env</type> <fileName>pig-env.cmd</fileName> <dictionaryName>pig-env</dictionaryName> </configFile> - <configFile> - <type>env</type> - <fileName>log4j.properties</fileName> - <dictionaryName>pig-log4j</dictionaryName> - </configFile> - <configFile> - <type>env</type> - <fileName>pig.properties</fileName> - <dictionaryName>pig-properties</dictionaryName> - </configFile> </configFiles> </component> </components> - <commandScript> - <script>scripts/service_check.py</script> - <scriptType>PYTHON</scriptType> - <timeout>300</timeout> - </commandScript> - - <requiredServices> - <service>YARN</service> - </requiredServices> - - <configuration-dependencies> - <config-type>pig-env</config-type> - <config-type>pig-log4j</config-type> - <config-type>pig-properties</config-type> - </configuration-dependencies> - </service> </services> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/af6f6e87/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/configuration/sqoop-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/configuration/sqoop-env.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/configuration/sqoop-env.xml index c3bcf7e..5fa7bcb 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/configuration/sqoop-env.xml +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/configuration/sqoop-env.xml @@ -21,11 +21,17 @@ --> <configuration> - <!-- sqoop-env.sh --> + <property> + <name>sqoop_user</name> + <deleted>true</deleted> + </property> + + <!-- sqoop-env.cmd --> <property> <name>content</name> <description>This is the jinja template for sqoop-env.cmd file</description> - <value>@echo off + <value> +@echo off :: Licensed to the Apache Software Foundation (ASF) under one or more :: contributor license agreements. See the NOTICE file distributed with :: this work for additional information regarding copyright ownership. http://git-wip-us.apache.org/repos/asf/ambari/blob/af6f6e87/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/metainfo.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/metainfo.xml index 13f9630..678a8a7 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/metainfo.xml @@ -20,38 +20,11 @@ <services> <service> <name>SQOOP</name> - <displayName>Sqoop</displayName> - <comment>Tool for transferring bulk data between Apache Hadoop and - structured data stores such as relational databases - </comment> - <version>1.4.4.2.0</version> - + <extends>common-services/SQOOP/1.4.4.2.0</extends> + <version>1.4.4.2.1.1.0</version> <components> <component> <name>SQOOP</name> - <displayName>Sqoop</displayName> - <category>CLIENT</category> - <cardinality>1+</cardinality> - <dependencies> - <dependency> - <name>HDFS/HDFS_CLIENT</name> - <scope>host</scope> - <auto-deploy> - <enabled>true</enabled> - </auto-deploy> - </dependency> - <dependency> - <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name> - <scope>host</scope> - <auto-deploy> - <enabled>true</enabled> - </auto-deploy> - </dependency> - </dependencies> - <commandScript> - <script>scripts/sqoop_client.py</script> - <scriptType>PYTHON</scriptType> - </commandScript> <configFiles> <configFile> <type>env</type> @@ -61,20 +34,6 @@ </configFiles> </component> </components> - - <commandScript> - <script>scripts/service_check.py</script> - <scriptType>PYTHON</scriptType> - <timeout>300</timeout> - </commandScript> - - <requiredServices> - <service>HDFS</service> - </requiredServices> - - <configuration-dependencies> - <config-type>sqoop-env</config-type> - </configuration-dependencies> </service> </services> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/af6f6e87/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/configuration/storm-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/configuration/storm-env.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/configuration/storm-env.xml new file mode 100644 index 0000000..4d9aae6 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/configuration/storm-env.xml @@ -0,0 +1,47 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + <property> + <name>storm_user</name> + <deleted>true</deleted> + </property> + + <property> + <name>storm_log_dir</name> + <value>c:\hadoop\logs\storm</value> + <description></description> + </property> + <property> + <name>storm_pid_dir</name> + <value>c:\hadoop\run\storm</value> + <description></description> + </property> + + <!-- storm-env.cmd --> + <property> + <name>content</name> + <description>This is the jinja template for storm-env.cmd file</description> + <value> + </value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/af6f6e87/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/configuration/storm-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/configuration/storm-site.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/configuration/storm-site.xml index 5a23314..c0143f1 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/configuration/storm-site.xml +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/configuration/storm-site.xml @@ -22,627 +22,15 @@ <configuration supports_final="true"> <property> - <name>storm.zookeeper.servers</name> - <value>['localhost']</value> - <description>A list of hosts of ZooKeeper servers used to manage the cluster.</description> - </property> - <property> - <name>nimbus.host</name> - <value>localhost</value> - <description>The host that the master server is running on.</description> - </property> - <property> <name>storm.local.dir</name> - <value>c:\\hdpdata</value> + <value>c:\hadoop\storm</value> <description>A directory on the local filesystem used by Storm for any local filesystem usage it needs. The directory must exist and the Storm daemons must have permission to read/write from this location.</description> </property> <property> - <name>logviewer.port</name> - <value>8081</value> - <description>HTTP UI port for log viewer.</description> - </property> - <property> - <name>storm.messaging.transport</name> - <value>backtype.storm.messaging.netty.Context</value> - <description>The transporter for communication among Storm tasks.</description> - </property> - <property> - <name>storm.messaging.netty.buffer_size</name> - <value>16384</value> - <description>Netty based messaging: The buffer size for send/recv buffer.</description> - </property> - <property> - <name>storm.messaging.netty.max_retries</name> - <value>10</value> - <description>Netty based messaging: The max # of retries that a peer will perform when a remote is not accessible.</description> - </property> - <property> - <name>storm.messaging.netty.max_wait_ms</name> - <value>5000</value> - <description>Netty based messaging: The max # of milliseconds that a peer will wait.</description> - </property> - <property> - <name>storm.messaging.netty.min_wait_ms</name> - <value>1000</value> - <description>Netty based messaging: The min # of milliseconds that a peer will wait.</description> - </property> - <property> - <name>ui.port</name> - <value>8772</value> - <description>Storm UI binds to this port.</description> - </property> - <property> - <name>java.library.path</name> - <value>/usr/local/lib:/opt/local/lib:/usr/lib</value> - <description>This value is passed to spawned JVMs (e.g., Nimbus, Supervisor, and Workers) - for the java.library.path value. java.library.path tells the JVM where - to look for native libraries. It is necessary to set this config correctly since - Storm uses the ZeroMQ and JZMQ native libs. - </description> - </property> - <property> - <name>storm.zookeeper.servers</name> - <value>['localhost']</value> - <description>A list of hosts of ZooKeeper servers used to manage the cluster.</description> - </property> - <property> - <name>storm.zookeeper.port</name> - <value>2181</value> - <description>The port Storm will use to connect to each of the ZooKeeper servers.</description> - </property> - <property> - <name>storm.zookeeper.root</name> - <value>/storm</value> - <description>The root location at which Storm stores data in ZooKeeper.</description> - </property> - <property> - <name>storm.zookeeper.session.timeout</name> - <value>20000</value> - <description>The session timeout for clients to ZooKeeper.</description> - </property> - <property> - <name>storm.zookeeper.connection.timeout</name> - <value>15000</value> - <description>The connection timeout for clients to ZooKeeper.</description> - </property> - <property> - <name>storm.zookeeper.retry.times</name> - <value>5</value> - <description>The number of times to retry a Zookeeper operation.</description> - </property> - <property> - <name>storm.zookeeper.retry.interval</name> - <value>1000</value> - <description>The interval between retries of a Zookeeper operation.</description> - </property> - <property> - <name>storm.zookeeper.retry.intervalceiling.millis</name> - <value>30000</value> - <description>The ceiling of the interval between retries of a Zookeeper operation.</description> - </property> - <property> - <name>storm.cluster.mode</name> - <value>distributed</value> - <description>The mode this Storm cluster is running in. Either "distributed" or "local".</description> - </property> - <property> - <name>storm.local.mode.zmq</name> - <value>false</value> - <description>Whether or not to use ZeroMQ for messaging in local mode. If this is set - to false, then Storm will use a pure-Java messaging system. The purpose - of this flag is to make it easy to run Storm in local mode by eliminating - the need for native dependencies, which can be difficult to install. - </description> - </property> - <property> - <name>storm.thrift.transport</name> - <value>backtype.storm.security.auth.SimpleTransportPlugin</value> - <description>The transport plug-in for Thrift client/server communication.</description> - </property> - <property> - <name>storm.messaging.transport</name> - <value>backtype.storm.messaging.netty.Context</value> - <description>The transporter for communication among Storm tasks.</description> - </property> - <property> - <name>nimbus.host</name> - <value>localhost</value> - <description>The host that the master server is running on.</description> - </property> - <property> - <name>nimbus.thrift.port</name> - <value>6627</value> - <description>Which port the Thrift interface of Nimbus should run on. Clients should - connect to this port to upload jars and submit topologies. - </description> - </property> - <property> - <name>nimbus.thrift.max_buffer_size</name> - <value>1048576</value> - <description>The maximum buffer size thrift should use when reading messages.</description> - </property> - <property> - <name>nimbus.childopts</name> - <value>-Xmx1024m</value> - <description>This parameter is used by the storm-deploy project to configure the jvm options for the nimbus - daemon. - </description> - </property> - <property> - <name>nimbus.task.timeout.secs</name> - <value>30</value> - <description>How long without heartbeating a task can go before nimbus will consider the task dead and reassign it - to another location. - </description> - </property> - <property> - <name>nimbus.supervisor.timeout.secs</name> - <value>60</value> - <description>How long before a supervisor can go without heartbeating before nimbus considers it dead and stops - assigning new work to it. - </description> - </property> - <property> - <name>nimbus.monitor.freq.secs</name> - <value>10</value> - <description> - How often nimbus should wake up to check heartbeats and do reassignments. Note - that if a machine ever goes down Nimbus will immediately wake up and take action. - This parameter is for checking for failures when there's no explicit event like that occuring. - </description> - </property> - <property> - <name>nimbus.cleanup.inbox.freq.secs</name> - <value>600</value> - <description>How often nimbus should wake the cleanup thread to clean the inbox.</description> - </property> - <property> - <name>nimbus.inbox.jar.expiration.secs</name> - <value>3600</value> - <description> - The length of time a jar file lives in the inbox before being deleted by the cleanup thread. - - Probably keep this value greater than or equal to NIMBUS_CLEANUP_INBOX_JAR_EXPIRATION_SECS. - Note that the time it takes to delete an inbox jar file is going to be somewhat more than - NIMBUS_CLEANUP_INBOX_JAR_EXPIRATION_SECS (depending on how often NIMBUS_CLEANUP_FREQ_SECS is set to). - </description> - </property> - <property> - <name>nimbus.task.launch.secs</name> - <value>120</value> - <description>A special timeout used when a task is initially launched. During launch, this is the timeout - used until the first heartbeat, overriding nimbus.task.timeout.secs. - </description> - </property> - <property> - <name>nimbus.reassign</name> - <value>true</value> - <description>Whether or not nimbus should reassign tasks if it detects that a task goes down. - Defaults to true, and it's not recommended to change this value. - </description> - </property> - <property> - <name>nimbus.file.copy.expiration.secs</name> - <value>600</value> - <description>During upload/download with the master, how long an upload or download connection is idle - before nimbus considers it dead and drops the connection. - </description> - </property> - <property> - <name>nimbus.topology.validator</name> - <value>backtype.storm.nimbus.DefaultTopologyValidator</value> - <description>A custom class that implements ITopologyValidator that is run whenever a - topology is submitted. Can be used to provide business-specific logic for - whether topologies are allowed to run or not. - </description> - </property> - <property> - <name>ui.childopts</name> - <value>-Xmx768m -Djava.security.auth.login.config=/etc/storm/conf/storm_jaas.conf</value> - <description>Childopts for Storm UI Java process.</description> - </property> - <property> - <name>logviewer.childopts</name> - <value>-Xmx128m</value> - <description>Childopts for log viewer java process.</description> - </property> - <property> - <name>logviewer.appender.name</name> - <value>A1</value> - <description>Appender name used by log viewer to determine log directory.</description> - </property> - <property> - <name>drpc.port</name> - <value>3772</value> - <description>This port is used by Storm DRPC for receiving DPRC requests from clients.</description> - </property> - <property> - <name>drpc.worker.threads</name> - <value>64</value> - <description>DRPC thrift server worker threads.</description> - </property> - <property> - <name>drpc.queue.size</name> - <value>128</value> - <description>DRPC thrift server queue size.</description> - </property> - <property> - <name>drpc.invocations.port</name> - <value>3773</value> - <description>This port on Storm DRPC is used by DRPC topologies to receive function invocations and send results - back. - </description> - </property> - <property> - <name>drpc.request.timeout.secs</name> - <value>600</value> - <description>The timeout on DRPC requests within the DRPC server. Defaults to 10 minutes. Note that requests can - also - timeout based on the socket timeout on the DRPC client, and separately based on the topology message - timeout for the topology implementing the DRPC function. - </description> - </property> - <property> - <name>drpc.childopts</name> - <value>-Xmx768m</value> - <description>Childopts for Storm DRPC Java process.</description> - </property> - <property> - <name>transactional.zookeeper.root</name> - <value>/transactional</value> - <description>The root directory in ZooKeeper for metadata about TransactionalSpouts.</description> - </property> - <property> - <name>transactional.zookeeper.servers</name> - <value>null</value> - <description>The list of zookeeper servers in which to keep the transactional state. If null (which is default), - will use storm.zookeeper.servers - </description> - </property> - <property> - <name>transactional.zookeeper.port</name> - <value>null</value> - <description>The port to use to connect to the transactional zookeeper servers. If null (which is default), - will use storm.zookeeper.port - </description> - </property> - <property> - <name>supervisor.slots.ports</name> - <value>[6700, 6701]</value> - <description>A list of ports that can run workers on this supervisor. Each worker uses one port, and - the supervisor will only run one worker per port. Use this configuration to tune - how many workers run on each machine. - </description> - </property> - <property> - <name>supervisor.childopts</name> - <value>-Xmx256m</value> - <description>This parameter is used by the storm-deploy project to configure the jvm options for the supervisor - daemon. - </description> - </property> - <property> - <name>supervisor.worker.start.timeout.secs</name> - <value>120</value> - <description>How long a worker can go without heartbeating during the initial launch before - the supervisor tries to restart the worker process. This value override - supervisor.worker.timeout.secs during launch because there is additional - overhead to starting and configuring the JVM on launch. - </description> - </property> - <property> - <name>supervisor.worker.timeout.secs</name> - <value>30</value> - <description>How long a worker can go without heartbeating before the supervisor tries to restart the worker - process. - </description> - </property> - <property> - <name>supervisor.monitor.frequency.secs</name> - <value>3</value> - <description>How often the supervisor checks the worker heartbeats to see if any of them need to be restarted. - </description> - </property> - <property> - <name>supervisor.heartbeat.frequency.secs</name> - <value>5</value> - <description>How often the supervisor sends a heartbeat to the master.</description> - </property> - <property> - <name>worker.childopts</name> - <value>-Xmx768m</value> - <description>The jvm opts provided to workers launched by this supervisor. All \"%ID%\" substrings are replaced with - an identifier for this worker. - </description> - </property> - <property> - <name>worker.heartbeat.frequency.secs</name> - <value>1</value> - <description>How often this worker should heartbeat to the supervisor.</description> - </property> - <property> - <name>task.heartbeat.frequency.secs</name> - <value>3</value> - <description>How often a task should heartbeat its status to the master.</description> - </property> - <property> - <name>task.refresh.poll.secs</name> - <value>10</value> - <description>How often a task should sync its connections with other tasks (if a task is - reassigned, the other tasks sending messages to it need to refresh their connections). - In general though, when a reassignment happens other tasks will be notified - almost immediately. This configuration is here just in case that notification doesn't - come through. - </description> - </property> - <property> - <name>zmq.threads</name> - <value>1</value> - <description>The number of threads that should be used by the zeromq context in each worker process.</description> - </property> - <property> - <name>zmq.linger.millis</name> - <value>5000</value> - <description>How long a connection should retry sending messages to a target host when - the connection is closed. This is an advanced configuration and can almost - certainly be ignored. - </description> - </property> - <property> - <name>zmq.hwm</name> - <value>0</value> - <description>The high water for the ZeroMQ push sockets used for networking. Use this config to prevent buffer - explosion - on the networking layer. - </description> - </property> - <property> - <name>storm.messaging.netty.server_worker_threads</name> - <value>1</value> - <description>Netty based messaging: The # of worker threads for the server.</description> - </property> - <property> - <name>storm.messaging.netty.client_worker_threads</name> - <value>1</value> - <description>Netty based messaging: The # of worker threads for the client.</description> - </property> - <property> - <name>topology.enable.message.timeouts</name> - <value>true</value> - <description>True if Storm should timeout messages or not. Defaults to true. This is meant to be used - in unit tests to prevent tuples from being accidentally timed out during the test. - </description> - </property> - <property> - <name>topology.debug</name> - <value>false</value> - <description>When set to true, Storm will log every message that's emitted.</description> - </property> - <property> - <name>topology.optimize</name> - <value>true</value> - <description>Whether or not the master should optimize topologies by running multiple tasks in a single thread where - appropriate. - </description> - </property> - <property> - <name>topology.workers</name> - <value>1</value> - <description>How many processes should be spawned around the cluster to execute this - topology. Each process will execute some number of tasks as threads within - them. This parameter should be used in conjunction with the parallelism hints - on each component in the topology to tune the performance of a topology. - </description> - </property> - <property> - <name>topology.acker.executors</name> - <value>null</value> - <description>How many executors to spawn for ackers. - - If this is set to 0, then Storm will immediately ack tuples as soon - as they come off the spout, effectively disabling reliability. - </description> - </property> - <property> - <name>topology.message.timeout.secs</name> - <value>30</value> - <description>The maximum amount of time given to the topology to fully process a message - emitted by a spout. If the message is not acked within this time frame, Storm - will fail the message on the spout. Some spouts implementations will then replay - the message at a later time. - </description> - </property> - <property> - <name>topology.skip.missing.kryo.registrations</name> - <value>false</value> - <description>Whether or not Storm should skip the loading of kryo registrations for which it - does not know the class or have the serializer implementation. Otherwise, the task will - fail to load and will throw an error at runtime. The use case of this is if you want to - declare your serializations on the storm.yaml files on the cluster rather than every single - time you submit a topology. Different applications may use different serializations and so - a single application may not have the code for the other serializers used by other apps. - By setting this config to true, Storm will ignore that it doesn't have those other serializations - rather than throw an error. - </description> - </property> - <property> - <name>topology.max.task.parallelism</name> - <value>null</value> - <description>The maximum parallelism allowed for a component in this topology. This configuration is - typically used in testing to limit the number of threads spawned in local mode. - </description> - </property> - <property> - <name>topology.max.spout.pending</name> - <value>null</value> - <description>The maximum number of tuples that can be pending on a spout task at any given time. - This config applies to individual tasks, not to spouts or topologies as a whole. - - A pending tuple is one that has been emitted from a spout but has not been acked or failed yet. - Note that this config parameter has no effect for unreliable spouts that don't tag - their tuples with a message id. - </description> - </property> - <property> - <name>topology.state.synchronization.timeout.secs</name> - <value>60</value> - <description>The maximum amount of time a component gives a source of state to synchronize before it requests - synchronization again. - </description> - </property> - <property> - <name>topology.stats.sample.rate</name> - <value>0.05</value> - <description>The percentage of tuples to sample to produce stats for a task.</description> - </property> - <property> - <name>topology.builtin.metrics.bucket.size.secs</name> - <value>60</value> - <description>The time period that builtin metrics data in bucketed into.</description> - </property> - <property> - <name>topology.fall.back.on.java.serialization</name> - <value>true</value> - <description>Whether or not to use Java serialization in a topology.</description> - </property> - <property> - <name>topology.worker.childopts</name> - <value>null</value> - <description>Topology-specific options for the worker child process. This is used in addition to WORKER_CHILDOPTS. - </description> - </property> - <property> - <name>topology.executor.receive.buffer.size</name> - <value>1024</value> - <description>The size of the Disruptor receive queue for each executor. Must be a power of 2.</description> - </property> - <property> - <name>topology.executor.send.buffer.size</name> - <value>1024</value> - <description>The size of the Disruptor send queue for each executor. Must be a power of 2.</description> - </property> - <property> - <name>topology.receiver.buffer.size</name> - <value>8</value> - <description>The maximum number of messages to batch from the thread receiving off the network to the - executor queues. Must be a power of 2. - </description> - </property> - <property> - <name>topology.transfer.buffer.size</name> - <value>1024</value> - <description>The size of the Disruptor transfer queue for each worker.</description> - </property> - <property> - <name>topology.tick.tuple.freq.secs</name> - <value>null</value> - <description>How often a tick tuple from the "__system" component and "__tick" stream should be sent - to tasks. Meant to be used as a component-specific configuration. - </description> - </property> - <property> - <name>topology.worker.shared.thread.pool.size</name> - <value>4</value> - <description>The size of the shared thread pool for worker tasks to make use of. The thread pool can be accessed - via the TopologyContext. - </description> - </property> - <property> - <name>topology.disruptor.wait.strategy</name> - <value>com.lmax.disruptor.BlockingWaitStrategy</value> - <description>Configure the wait strategy used for internal queuing. Can be used to tradeoff latency - vs. throughput. - </description> - </property> - <property> - <name>topology.executor.send.buffer.size</name> - <value>1024</value> - <description>The size of the Disruptor send queue for each executor. Must be a power of 2.</description> - </property> - <property> - <name>topology.receiver.buffer.size</name> - <value>8</value> - <description>The maximum number of messages to batch from the thread receiving off the network to the - executor queues. Must be a power of 2. - </description> - </property> - <property> - <name>topology.transfer.buffer.size</name> - <value>1024</value> - <description>The size of the Disruptor transfer queue for each worker.</description> - </property> - <property> - <name>topology.tick.tuple.freq.secs</name> - <value>null</value> - <description>How often a tick tuple from the "__system" component and "__tick" stream should be sent - to tasks. Meant to be used as a component-specific configuration. - </description> - </property> - <property> - <name>topology.worker.shared.thread.pool.size</name> - <value>4</value> - <description>The size of the shared thread pool for worker tasks to make use of. The thread pool can be accessed - via the TopologyContext. - </description> - </property> - <property> - <name>topology.spout.wait.strategy</name> - <value>backtype.storm.spout.SleepSpoutWaitStrategy</value> - <description>A class that implements a strategy for what to do when a spout needs to wait. Waiting is - triggered in one of two conditions: - - 1. nextTuple emits no tuples - 2. The spout has hit maxSpoutPending and can't emit any more tuples - </description> - </property> - <property> - <name>topology.sleep.spout.wait.strategy.time.ms</name> - <value>1</value> - <description>The amount of milliseconds the SleepEmptyEmitStrategy should sleep for.</description> - </property> - <property> - <name>topology.error.throttle.interval.secs</name> - <value>10</value> - <description>The interval in seconds to use for determining whether to throttle error reported to Zookeeper. For - example, - an interval of 10 seconds with topology.max.error.report.per.interval set to 5 will only allow 5 errors to be - reported to Zookeeper per task for every 10 second interval of time. - </description> - </property> - <property> - <name>topology.max.error.report.per.interval</name> - <value>5</value> - <description>The interval in seconds to use for determining whether to throttle error reported to Zookeeper. For - example, - an interval of 10 seconds with topology.max.error.report.per.interval set to 5 will only allow 5 errors to be - reported to Zookeeper per task for every 10 second interval of time. - </description> - </property> - <property> - <name>topology.kryo.factory</name> - <value>backtype.storm.serialization.DefaultKryoFactory</value> - <description>Class that specifies how to create a Kryo instance for serialization. Storm will then apply - topology.kryo.register and topology.kryo.decorators on top of this. The default implementation - implements topology.fall.back.on.java.serialization and turns references off. - </description> - </property> - <property> - <name>topology.tuple.serializer</name> - <value>backtype.storm.serialization.types.ListDelegateSerializer</value> - <description>The serializer class for ListDelegate (tuple payload). - The default serializer will be ListDelegateSerializer - </description> - </property> - <property> - <name>topology.trident.batch.emit.interval.millis</name> - <value>500</value> - <description>How often a batch can be emitted in a Trident topology.</description> - </property> - <property> <name>dev.zookeeper.path</name> - <value>/tmp/dev-storm-zookeeper</value> + <value>c:\hadoop\temp\dev-storm-zookeeper</value> <description>The path to use as the zookeeper dir when running a zookeeper server via "storm dev-zookeeper". This zookeeper instance is only intended for development; it is not a production grade zookeeper setup. http://git-wip-us.apache.org/repos/asf/ambari/blob/af6f6e87/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/metainfo.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/metainfo.xml index 9df2aa8..76022cc 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/metainfo.xml @@ -21,72 +21,18 @@ <services> <service> <name>STORM</name> - <displayName>Storm</displayName> - <comment>Apache Hadoop Stream processing framework</comment> - <version>0.9.1.2.1</version> + <extends>common-services/STORM/0.9.1.2.1</extends> + <version>0.9.1.2.1.1.0</version> <components> - <component> - <name>NIMBUS</name> - <displayName>Nimbus</displayName> - <category>MASTER</category> - <cardinality>1</cardinality> - <dependencies> - <dependency> - <name>ZOOKEEPER/ZOOKEEPER_SERVER</name> - <scope>cluster</scope> - <auto-deploy> - <enabled>true</enabled> - </auto-deploy> - </dependency> - </dependencies> - <commandScript> - <script>scripts/nimbus.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> + <name>STORM_REST_API</name> + <deleted>true</deleted> </component> - - <component> - <name>SUPERVISOR</name> - <displayName>Supervisor</displayName> - <category>SLAVE</category> - <cardinality>1+</cardinality> - <commandScript> - <script>scripts/supervisor.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - </component> - <component> - <name>STORM_UI_SERVER</name> - <displayName>Storm UI Server</displayName> - <category>MASTER</category> - <cardinality>1</cardinality> - <commandScript> - <script>scripts/ui_server.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> + <name>DRPC_SERVER</name> + <deleted>true</deleted> </component> - </components> - - <commandScript> - <script>scripts/service_check.py</script> - <scriptType>PYTHON</scriptType> - <timeout>300</timeout> - </commandScript> - - <requiredServices> - <service>ZOOKEEPER</service> - </requiredServices> - - <configuration-dependencies> - <config-type>storm-site</config-type> - <config-type>storm-env</config-type> - </configuration-dependencies> </service> </services> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/af6f6e87/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/configuration/tez-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/configuration/tez-env.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/configuration/tez-env.xml new file mode 100644 index 0000000..fd7e7b0 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/configuration/tez-env.xml @@ -0,0 +1,36 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + <property> + <name>tez_user</name> + <deleted>true</deleted> + </property> + + <!-- tez-env.cmd --> + <property> + <name>content</name> + <description>This is the jinja template for tez-env.cmd file</description> + <value> + </value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/af6f6e87/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/configuration/tez-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/configuration/tez-site.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/configuration/tez-site.xml index 42eaa45..ac788bf 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/configuration/tez-site.xml +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/configuration/tez-site.xml @@ -20,199 +20,8 @@ <configuration supports_final="true"> <property> - <name>tez.lib.uris</name> - <value>hdfs:///apps/tez/,hdfs:///apps/tez/lib/</value> - <!-- TODO:Remove this value>${fs.default.name}/apps/tez/,${fs.default.name}/apps/tez/lib</value--> - <description>The location of the Tez libraries which will be localized for DAGs</description> - </property> - - <property> - <name>tez.am.log.level</name> - <value>INFO</value> - <description>Root Logging level passed to the Tez app master</description> - </property> - - <property> <name>tez.staging-dir</name> - <value>/tmp/${user.name}/staging</value> + <value>c:\hadoop\temp\${user.name}\staging</value> <description>The staging dir used while submitting DAGs</description> </property> - - <property> - <name>tez.am.resource.memory.mb</name> - <value>1536</value> - <!-- TODO: Value needs to be set based on YARN configuration - similar to the way the MR AM size is set, 1.5 times the MR AM size --> - <description>The amount of memory to be used by the AppMaster</description> - </property> - - <property> - <name>tez.am.java.opts</name> - <value>-server -Xmx1024m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC</value> - <!-- TODO: Xmx value needs to be set based on tez.am.resource.memory.mb - typically at least 512MB less. Also this could have some additional hardware specific settings if Ambari supports it --> - <description>Java options for the Tez AppMaster process</description> - </property> - <!-- tez picks the java opts from yarn.app.mapreduce.am.command-opts for MR tasks. Likewise for the AM memory MB --> - - <property> - <name>tez.am.shuffle-vertex-manager.min-src-fraction</name> - <value>0.2</value> - <description>In case of a ScatterGather connection, the fraction of source tasks which should - complete before tasks for the current vertex are schedule - </description> - </property> - - <property> - <name>tez.am.shuffle-vertex-manager.max-src-fraction</name> - <value>0.4</value> - <description>In case of a ScatterGather connection, once this fraction of source tasks have - completed, all tasks on the current vertex can be scheduled. Number of tasks ready for - scheduling on the current vertex scales linearly between min-fraction and max-fraction - </description> - </property> - - <property> - <name>tez.am.am-rm.heartbeat.interval-ms.max</name> - <value>250</value> - <description>The maximum heartbeat interval between the AM and RM in milliseconds</description> - </property> - - <property> - <name>tez.am.grouping.split-waves</name> - <value>1.4</value> - <description>The multiplier for available queue capacity when determining number of tasks for - a Vertex. 1.4 with 100% queue available implies generating a number of tasks roughly equal - to 140% of the available containers on the queue - </description> - </property> - - <property> - <name>tez.am.grouping.min-size</name> - <value>16777216</value> - <description>Lower bound on the size (in bytes) of a grouped split, to avoid generating - too many splits - </description> - </property> - - <property> - <name>tez.am.grouping.max-size</name> - <value>1073741824</value> - <description>Upper bound on the size (in bytes) of a grouped split, to avoid generating - excessively large split - </description> - </property> - - <property> - <name>tez.am.container.reuse.enabled</name> - <value>true</value> - <description>Configuration to specify whether container should be reused</description> - </property> - - <property> - <name>tez.am.container.reuse.rack-fallback.enabled</name> - <value>true</value> - <description>Whether to reuse containers for rack local tasks. Active only if reuse is enabled - </description> - </property> - - <property> - <name>tez.am.container.reuse.non-local-fallback.enabled</name> - <value>true</value> - <description>Whether to reuse containers for non-local tasks. Active only if reuse is enabled - </description> - </property> - - <property> - <name>tez.am.container.session.delay-allocation-millis</name> - <value>10000</value> - <!-- TODO This value may change --> - <description>The amount of time to hold on to a container if no task can be assigned to - it immediately. Only active when reuse is enabled. Set to -1 to never release a container - in a session - </description> - </property> - - <property> - <name>tez.am.container.reuse.locality.delay-allocation-millis</name> - <value>250</value> - <description>The amount of time to wait before assigning a container to the next level of - locality. NODE -> RACK -> NON_LOCAL - </description> - </property> - - <property> - <name>tez.task.get-task.sleep.interval-ms.max</name> - <value>200</value> - <description>The maximum amount of time, in seconds, to wait before a task asks an AM for - another task - </description> - </property> - - <!-- Client Submission timeout value when submitting DAGs to a session --> - <property> - <name>tez.session.client.timeout.secs</name> - <value>180</value> - <description>Time (in seconds) to wait for AM to come up when trying to submit a DAG from - the client - </description> - </property> - - <property> - <name>tez.session.am.dag.submit.timeout.secs</name> - <value>300</value> - <description>Time (in seconds) for which the Tez AM should wait for a DAG to be submitted - before shutting down - </description> - </property> - - - <!-- Configuration for runtime components --> - - <!-- These properties can be set on a per edge basis by configuring the payload for each - edge independently. --> - - <property> - <name>tez.runtime.intermediate-output.should-compress</name> - <value>false</value> - <description>Whether intermediate output should be compressed or not</description> - </property> - - <property> - <name>tez.runtime.intermediate-output.compress.codec</name> - <value>org.apache.hadoop.io.compress.SnappyCodec</value> - <description>The coded to be used if compressing intermediate output. Only - applicable if tez.runtime.intermediate-output.should-compress is enabled. - </description> - </property> - - <property> - <name>tez.runtime.intermediate-input.is-compressed</name> - <value>false</value> - <description>Whether intermediate input is compressed</description> - </property> - - <property> - <name>tez.runtime.intermediate-input.compress.codec</name> - <value>org.apache.hadoop.io.compress.SnappyCodec</value> - <description>The coded to be used when reading intermediate compressed input. - Only applicable if tez.runtime.intermediate-input.is-compressed is enabled. - </description> - </property> - - <property> - <name>tez.runtime.job.counters.max</name> - <value>10000</value> - </property> - <property> - <name>tez.runtime.job.counters.groups.max</name> - <value>10000</value> - </property> - - <!-- Configuration for ATS integration --> - - <property> - <name>tez.yarn.ats.enabled</name> - <value>true</value> - <description>Whether to send history events to YARN Application Timeline Server</description> - </property> - </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/af6f6e87/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/metainfo.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/metainfo.xml index 1ca4d56..cbd6a30 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/metainfo.xml @@ -20,27 +20,13 @@ <services> <service> <name>TEZ</name> - <displayName>Tez</displayName> - <comment>Tez is the next generation Hadoop Query Processing framework written on top of YARN.</comment> - <version>0.4.0.2.1</version> + <extends>common-services/TEZ/0.4.0.2.1</extends> + <version>0.4.0.2.1.1.0</version> <components> <component> <name>TEZ_CLIENT</name> - <displayName>Tez Client</displayName> - <cardinality>1+</cardinality> - <category>CLIENT</category> - <commandScript> - <script>scripts/tez_client.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> <configFiles> <configFile> - <type>xml</type> - <fileName>tez-site.xml</fileName> - <dictionaryName>tez-site</dictionaryName> - </configFile> - <configFile> <type>env</type> <fileName>tez-env.cmd</fileName> <dictionaryName>tez-env</dictionaryName> @@ -48,16 +34,6 @@ </configFiles> </component> </components> - - <requiredServices> - <service>YARN</service> - </requiredServices> - - <configuration-dependencies> - <config-type>tez-site</config-type> - <config-type>tez-env</config-type> - </configuration-dependencies> - </service> </services> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/af6f6e87/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/metainfo.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/metainfo.xml index a1a4804..9c41776 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/metainfo.xml @@ -23,7 +23,7 @@ <name>YARN</name> <displayName>YARN</displayName> <comment>Apache Hadoop NextGen MapReduce (YARN)</comment> - <version>2.1.0.2.0</version> + <version>2.4.0.2.1.1.0</version> <components> <component> @@ -133,7 +133,7 @@ <name>MAPREDUCE2</name> <displayName>MapReduce2</displayName> <comment>Apache Hadoop NextGen MapReduce (YARN)</comment> - <version>2.1.0.2.0.6.0</version> + <version>2.4.0.2.1.1.0</version> <components> <component> <name>HISTORYSERVER</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/af6f6e87/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/configuration/zoo.cfg.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/configuration/zoo.cfg.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/configuration/zoo.cfg.xml index a28d3b2..244f75e 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/configuration/zoo.cfg.xml +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/configuration/zoo.cfg.xml @@ -22,28 +22,8 @@ <configuration> <property> - <name>tickTime</name> - <value>2000</value> - <description>The length of a single tick in milliseconds, which is the basic time unit used by ZooKeeper</description> - </property> - <property> - <name>initLimit</name> - <value>10</value> - <description>Ticks to allow for sync at Init.</description> - </property> - <property> - <name>syncLimit</name> - <value>5</value> - <description>Ticks to allow for sync at Runtime.</description> - </property> - <property> - <name>clientPort</name> - <value>2181</value> - <description>Port for running ZK Server.</description> - </property> - <property> <name>dataDir</name> - <value>C:\\\\hadoop\\\\zookeeper</value> + <value>c:\hadoop\zookeeper</value> <description>Data directory for ZooKeeper.</description> </property> </configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/af6f6e87/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/configuration/zookeeper-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/configuration/zookeeper-env.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/configuration/zookeeper-env.xml index a8964fa..64dd971 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/configuration/zookeeper-env.xml +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/configuration/zookeeper-env.xml @@ -23,25 +23,51 @@ <configuration> <property> <name>zk_user</name> - <value>zookeeper</value> - <description>ZooKeeper User.</description> + <deleted>true</deleted> + </property> + <property> + <name>zk_log_dir</name> + <value>c:\hadoop\log\zookeeper</value> + <description>ZooKeeper Log Dir</description> + </property> + <property> + <name>zk_pid_dir</name> + <value>c:\hadoop\run\zookeeper</value> + <description>ZooKeeper Pid Dir</description> </property> <!-- zookeeper-env.sh --> <property> <name>content</name> <description>zookeeper-env.sh content</description> <value> -export JAVA_HOME={{java64_home}} -export ZOO_LOG_DIR={{zk_log_dir}} -export ZOOPIDFILE={{zk_pid_file}} -export SERVER_JVMFLAGS={{zk_server_heapsize}} -export JAVA=$JAVA_HOME/bin/java -export CLASSPATH=$CLASSPATH:/usr/share/zookeeper/* +@echo off +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@rem Set environment variables here. + +@rem JVM parameters for both client and server +@rem JDK6 on Windows has a known bug for IPv6, use preferIPv4Stack unless JDK7. +set JVMFLAGS=-Djava.net.preferIPv4Stack=true + +@rem Client specific JVM parameters +@rem set CLIENT_JVMFLAGS= -{% if security_enabled %} -export SERVER_JVMFLAGS="$SERVER_JVMFLAGS -Djava.security.auth.login.config={{zk_server_jaas_file}}" -export CLIENT_JVMFLAGS="$CLIENT_JVMFLAGS -Djava.security.auth.login.config={{zk_client_jaas_file}}" -{% endif %} +@rem Server specific JVM parameters +@rem set SERVER_JVMFLAGS= </value> </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/af6f6e87/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/configuration/zookeeper-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/configuration/zookeeper-log4j.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/configuration/zookeeper-log4j.xml deleted file mode 100644 index 4dce6d1..0000000 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/configuration/zookeeper-log4j.xml +++ /dev/null @@ -1,100 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> - -<configuration supports_final="false"> - - <property> - <name>content</name> - <value> -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# -# - -# -# ZooKeeper Logging Configuration -# - -# DEFAULT: console appender only -log4j.rootLogger=INFO, CONSOLE - -# Example with rolling log file -#log4j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE - -# Example with rolling log file and tracing -#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE - -# -# Log INFO level and above messages to the console -# -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -log4j.appender.CONSOLE.Threshold=INFO -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n - -# -# Add ROLLINGFILE to rootLogger to get log file output -# Log DEBUG level and above messages to a log file -log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender -log4j.appender.ROLLINGFILE.Threshold=DEBUG -log4j.appender.ROLLINGFILE.File=zookeeper.log - -# Max log file size of 10MB -log4j.appender.ROLLINGFILE.MaxFileSize=10MB -# uncomment the next line to limit number of backup files -#log4j.appender.ROLLINGFILE.MaxBackupIndex=10 - -log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout -log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n - - -# -# Add TRACEFILE to rootLogger to get log file output -# Log DEBUG level and above messages to a log file -log4j.appender.TRACEFILE=org.apache.log4j.FileAppender -log4j.appender.TRACEFILE.Threshold=TRACE -log4j.appender.TRACEFILE.File=zookeeper_trace.log - -log4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout -### Notice we are including log4j's NDC here (%x) -log4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L][%x] - %m%n - </value> - </property> - -</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/af6f6e87/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/metainfo.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/metainfo.xml index daae2a7..afcc9cb 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/metainfo.xml @@ -20,59 +20,21 @@ <services> <service> <name>ZOOKEEPER</name> - <displayName>ZooKeeper</displayName> - <comment>Centralized service which provides highly reliable distributed coordination</comment> - <version>3.4.5.2.0</version> + <extends>common-services/ZOOKEEPER/3.4.5.2.0</extends> + <version>3.4.5.2.1.1.0</version> <components> <component> - <name>ZOOKEEPER_SERVER</name> - <displayName>ZooKeeper Server</displayName> - <category>MASTER</category> - <cardinality>1+</cardinality> - <commandScript> - <script>scripts/zookeeper_server.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - </component> - - <component> <name>ZOOKEEPER_CLIENT</name> - <displayName>ZooKeeper Client</displayName> - <category>CLIENT</category> - <cardinality>1+</cardinality> - <commandScript> - <script>scripts/zookeeper_client.py</script> - <scriptType>PYTHON</scriptType> - </commandScript> <configFiles> <configFile> <type>env</type> <fileName>zookeeper-env.cmd</fileName> <dictionaryName>zookeeper-env</dictionaryName> </configFile> - <configFile> - <type>env</type> - <fileName>log4j.properties</fileName> - <dictionaryName>zookeeper-log4j</dictionaryName> - </configFile> </configFiles> </component> </components> - - <commandScript> - <script>scripts/service_check.py</script> - <scriptType>PYTHON</scriptType> - <timeout>300</timeout> - </commandScript> - - <configuration-dependencies> - <config-type>zookeeper-log4j</config-type> - <config-type>zookeeper-env</config-type> - <config-type>zoo.cfg</config-type> - </configuration-dependencies> - <restartRequiredAfterChange>true</restartRequiredAfterChange> </service> </services> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/af6f6e87/ambari-web/app/utils/validator.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/utils/validator.js b/ambari-web/app/utils/validator.js index 3e84982..b7bfbf7 100644 --- a/ambari-web/app/utils/validator.js +++ b/ambari-web/app/utils/validator.js @@ -47,8 +47,8 @@ module.exports = { */ isValidDir: function(value){ var floatRegex = /^\/[0-9a-z]*/; - var winRegex = /^[a-z]:\\[0-9a-z]*/; - var winUrlRegex = /^file:\/\/\/[a-z]:\/[0-9a-z]*/; + var winRegex = /^[a-z]:\\[0-9a-zA-Z]*/; + var winUrlRegex = /^file:\/\/\/[a-zA-Z]:\/[0-9a-zA-Z]*/; var dirs = value.replace(/,/g,' ').trim().split(new RegExp("\\s+", "g")); for(var i = 0; i < dirs.length; i++){ if(!floatRegex.test(dirs[i]) && !winRegex.test(dirs[i]) && !winUrlRegex.test(dirs[i])){ @@ -65,9 +65,11 @@ module.exports = { */ isValidDataNodeDir: function(value) { var dirRegex = /^(\[[0-9a-zA-Z]+\])?(\/[0-9a-z]*)/; + var winRegex = /^(\[[0-9a-zA-Z]+\])?[a-zA-Z]:\\[0-9a-zA-Z]*/; + var winUrlRegex = /^(\[[0-9a-zA-Z]+\])?file:\/\/\/[a-zA-Z]:\/[0-9a-zA-Z]*/; var dirs = value.replace(/,/g,' ').trim().split(new RegExp("\\s+", "g")); for(var i = 0; i < dirs.length; i++){ - if(!dirRegex.test(dirs[i])){ + if(!dirRegex.test(dirs[i]) && !winRegex.test(dirs[i]) && !winUrlRegex.test(dirs[i])){ return false; } }
