Repository: ambari Updated Branches: refs/heads/branch-dev-logsearch b1be75f9a -> cb25daa33
http://git-wip-us.apache.org/repos/asf/ambari/blob/cb25daa3/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/solr-config.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/solr-config.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/solr-config.xml new file mode 100644 index 0000000..2505632 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/solr-config.xml @@ -0,0 +1,78 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + + <!-- Define configuration paramaters for service within Ambari: property name, default value and description (shown as help text) --> + + + <!-- <property> + <name>solr.cloudmode</name> + <value>true</value> + <description>Whether Solr should be started in Cloud mode</description> + </property> + + <property> + <name>solr.dir</name> + <value>/opt/solr</value> + <description>Path to Solr root. If HDPSearch is selected, it will be installed under /opt/lucidworks-hdpsearch/solr/</description> + </property> + + <property> + <name>solr.download.location</name> + <value>HDPSEARCH</value> + <description>Location to download Solr from (e.g. https://archive.apache.org/dist/lucene/solr/5.3.0/solr-5.3.0.tgz). Set this to HDPSEARCH to download HDPSearch from yum repo instead</description> + </property> --> + + + <property> + <name>solr.znode</name> + <value>/logsearch</value> + <description>Zookeeper znode</description> + </property> + + <property> + <name>solr.minmem</name> + <value>{smart_solr_memory}</value> + <description>Solr minimum heap size e.g.512m </description> + </property> + + <property> + <name>solr.maxmem</name> + <value>{smart_solr_memory}</value> + <description>Solr maximum heap size e.g. 512m</description> + </property> + + <property> + <name>logsearch.solr.datadir</name> + <value>{smart_solr_datadir}</value> + <description>Dir for Logsearch Solr data</description> + </property> + + <property> + <name>logsearch.solr.conf</name> + <value>/etc/logsearch/solr</value> + <description>Dir for Logsearch Solr configs</description> + </property> + + +</configuration> + http://git-wip-us.apache.org/repos/asf/ambari/blob/cb25daa3/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/solr-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/solr-env.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/solr-env.xml new file mode 100644 index 0000000..a1d13ba --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/solr-env.xml @@ -0,0 +1,156 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<!-- This is a special config file for properties used to monitor status of the service --> +<configuration> + + <property> + <name>solr_pid_dir</name> + <value>/var/run/solr/logsearch</value> + <description>Solr Process ID Directory</description> + </property> + + <property> + <name>solr.port</name> + <value>8886</value> + <description>Solr port</description> + </property> + + <property> + <name>solr.log.dir</name> + <value>/var/log/solr/logsearch</value> + <description>Directory for Solr logs</description> + </property> + + <property> + <name>solr.user</name> + <value>solr</value> + <property-type>USER</property-type> + <description>Solr user</description> + </property> + + <property> + <name>solr.group</name> + <value>solr</value> + <property-type>GROUP</property-type> + <description>Solr group</description> + </property> + + + <!-- solr.in.sh --> + + <property> + <name>content</name> + <description>This is the jinja template for solr.in.sh file</description> + <value> +# By default the script will use JAVA_HOME to determine which java +# to use, but you can set a specific path for Solr to use without +# affecting other Java applications on your server/workstation. +SOLR_JAVA_HOME={{java64_home}} + +# Increase Java Min/Max Heap as needed to support your indexing / query needs +SOLR_JAVA_MEM="-Xms{{solr_min_mem}} -Xmx{{solr_max_mem}}" + +# Enable verbose GC logging +GC_LOG_OPTS="-verbose:gc -XX:+PrintHeapAtGC -XX:+PrintGCDetails \ +-XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime" + +# These GC settings have shown to work well for a number of common Solr workloads +GC_TUNE="-XX:NewRatio=3 \ +-XX:SurvivorRatio=4 \ +-XX:TargetSurvivorRatio=90 \ +-XX:MaxTenuringThreshold=8 \ +-XX:+UseConcMarkSweepGC \ +-XX:+UseParNewGC \ +-XX:ConcGCThreads=4 -XX:ParallelGCThreads=4 \ +-XX:+CMSScavengeBeforeRemark \ +-XX:PretenureSizeThreshold=64m \ +-XX:+UseCMSInitiatingOccupancyOnly \ +-XX:CMSInitiatingOccupancyFraction=50 \ +-XX:CMSMaxAbortablePrecleanTime=6000 \ +-XX:+CMSParallelRemarkEnabled \ +-XX:+ParallelRefProcEnabled" + +# Set the ZooKeeper connection string if using an external ZooKeeper ensemble +# e.g. host1:2181,host2:2181/chroot +# Leave empty if not using SolrCloud +ZK_HOST="{{zookeeper_quorum}}{{solr_znode}}" + +# Set the ZooKeeper client timeout (for SolrCloud mode) +#ZK_CLIENT_TIMEOUT="15000" + +# By default the start script uses "localhost"; override the hostname here +# for production SolrCloud environments to control the hostname exposed to cluster state +#SOLR_HOST="192.168.1.1" + +# By default the start script uses UTC; override the timezone if needed +#SOLR_TIMEZONE="UTC" + +# Set to true to activate the JMX RMI connector to allow remote JMX client applications +# to monitor the JVM hosting Solr; set to "false" to disable that behavior +# (false is recommended in production environments) +ENABLE_REMOTE_JMX_OPTS="false" + +# The script will use SOLR_PORT+10000 for the RMI_PORT or you can set it here +# RMI_PORT=18983 + +# Anything you add to the SOLR_OPTS variable will be included in the java +# start command line as-is, in ADDITION to other options. If you specify the +# -a option on start script, those options will be appended as well. Examples: +#SOLR_OPTS="$SOLR_OPTS -Dsolr.autoSoftCommit.maxTime=3000" +#SOLR_OPTS="$SOLR_OPTS -Dsolr.autoCommit.maxTime=60000" +#SOLR_OPTS="$SOLR_OPTS -Dsolr.clustering.enabled=true" + +# Location where the bin/solr script will save PID files for running instances +# If not set, the script will create PID files in $SOLR_TIP/bin +SOLR_PID_DIR={{solr_piddir}} + +# Path to a directory where Solr creates index files, the specified directory +# must contain a solr.xml; by default, Solr will use server/solr +SOLR_HOME={{logsearch_solr_datadir}} + +# Solr provides a default Log4J configuration properties file in server/resources +# however, you may want to customize the log settings and file appender location +# so you can point the script to use a different log4j.properties file +LOG4J_PROPS={{logsearch_solr_datadir}}/resources/log4j.properties + +# Location where Solr should write logs to; should agree with the file appender +# settings in server/resources/log4j.properties +SOLR_LOGS_DIR={{solr_log_dir}} + +# Sets the port Solr binds to, default is 8983 +SOLR_PORT={{solr_port}} + +# Uncomment to set SSL-related system properties +# Be sure to update the paths to the correct keystore for your environment +#SOLR_SSL_OPTS="-Djavax.net.ssl.keyStore=etc/solr-ssl.keystore.jks \ +#-Djavax.net.ssl.keyStorePassword=secret \ +#-Djavax.net.ssl.trustStore=etc/solr-ssl.keystore.jks \ +#-Djavax.net.ssl.trustStorePassword=secret" + +# Uncomment to set a specific SSL port (-Djetty.ssl.port=N); if not set +# and you are using SSL, then the start script will use SOLR_PORT for the SSL port +#SOLR_SSL_PORT= + </value> + </property> + + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/cb25daa3/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/solr-log4j-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/solr-log4j-env.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/solr-log4j-env.xml new file mode 100644 index 0000000..0c02cf1 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/solr-log4j-env.xml @@ -0,0 +1,59 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + + + + <property> + <name>content</name> + <description>This is the jinja template for log4j.properties</description> + <value> +# Logging level +solr.log={{solr_log_dir}} +#log4j.rootLogger=INFO, file, CONSOLE +log4j.rootLogger=INFO, file + +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender + +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x [%X{collection} %X{shard} %X{replica} %X{core}] \u2013 %m%n + +#- size rotation with log cleanup. +log4j.appender.file=org.apache.log4j.RollingFileAppender +log4j.appender.file.MaxFileSize=10MB +log4j.appender.file.MaxBackupIndex=9 + +#- File to log to and log format +log4j.appender.file.File=${solr.log}/solr.log +log4j.appender.file.layout=org.apache.log4j.PatternLayout +log4j.appender.file.layout.ConversionPattern=%d{ISO8601} [%t] %-5p [%X{collection} %X{shard} %X{replica} %X{core}] %C (%F:%L) - %m%n + +log4j.logger.org.apache.zookeeper=WARN +log4j.logger.org.apache.hadoop=WARN + +# set to INFO to enable infostream log messages +log4j.logger.org.apache.solr.update.LoggingInfoStream=OFF + </value> + </property> + + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/cb25daa3/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/solr-sh.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/solr-sh.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/solr-sh.xml new file mode 100644 index 0000000..386bb54 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/solr-sh.xml @@ -0,0 +1,51 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<!-- This creates the solr.sh wrapper to include SOLR_INCLUDE in command line --> +<configuration> + <!-- solr.sh --> + + <property> + <name>content</name> + <description>This is the jinja template for solr.sh file</description> + <value> +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#This script is a wrapper for calling bin/solr but using custom properties +SOLR_INCLUDE={{logsearch_solr_conf}}/solr.in.sh {{solr_bindir}}/solr $* + </value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/cb25daa3/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/solr-xml-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/solr-xml-env.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/solr-xml-env.xml new file mode 100644 index 0000000..b8ecf9d --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/solr-xml-env.xml @@ -0,0 +1,42 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + + + <property> + <name>content</name> + <description>This is the jinja template for solr.xml file</description> + <value> +<solr> + <solrcloud> + <str name="host">${host:}</str> + <int name="hostPort">${jetty.port:}</int> + <str name="hostContext">${hostContext:solr}</str> + <int name="zkClientTimeout">${zkClientTimeout:15000}</int> + <bool name="genericCoreNodeNames">${genericCoreNodeNames:true}</bool> + </solrcloud> +</solr> + </value> + </property> + + +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/cb25daa3/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/solr-zoo-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/solr-zoo-env.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/solr-zoo-env.xml new file mode 100644 index 0000000..0a1edf8 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/solr-zoo-env.xml @@ -0,0 +1,50 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + + + + <property> + <name>content</name> + <description>This is the jinja template for /opt/logsearch_solr/data/zoo.cfg</description> + <value> +# The number of milliseconds of each tick +tickTime=2000 +# The number of ticks that the initial +# synchronization phase can take +initLimit=10 +# The number of ticks that can pass between +# sending a request and getting an acknowledgement +syncLimit=5 + +# the directory where the snapshot is stored. +# dataDir=/opt/zookeeper/data +# NOTE: Solr defaults the dataDir to $solrHome/zoo_data + +# the port at which the clients will connect +# clientPort=2181 +# NOTE: Solr sets this based on zkRun / zkHost params + </value> + </property> + + +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/cb25daa3/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/kerberos.json b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/kerberos.json new file mode 100644 index 0000000..8d49d90 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/kerberos.json @@ -0,0 +1,17 @@ +{ + "services": [ + { + "name": "LOGSEARCH", + "identities": [ + { + "name": "/smokeuser" + } + ], + "components": [ + { + "name": "LOGSEARCH_MASTER" + } + ] + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/cb25daa3/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml new file mode 100644 index 0000000..8d7f665 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml @@ -0,0 +1,113 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>LOGSEARCH</name> + <displayName>Log Search</displayName> + <comment>Log aggregation, analysis, and visualization for Hadoop</comment> + <version>0.5.0</version> + + + <components> + + <component> + <name>LOGSEARCH_MASTER</name> + <timelineAppid>logsearch</timelineAppid> + <displayName>Log Search</displayName> + <category>MASTER</category> + <cardinality>1</cardinality> + <commandScript> + <script>scripts/logsearch.py</script> + <scriptType>PYTHON</scriptType> + <timeout>5000</timeout> + </commandScript> + <customCommands> + <customCommand> + <name>UPDATE_LOGSEARCH</name> + <commandScript> + <script>scripts/logsearch.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </customCommand> + </customCommands> + + </component> + + <component> + <name>LOGSEARCH_SOLR</name> + <displayName>Solr</displayName> + <category>MASTER</category> + <cardinality>1+</cardinality> + <commandScript> + <script>scripts/solr.py</script> + <scriptType>PYTHON</scriptType> + <timeout>5000</timeout> + </commandScript> + </component> + + <component> + <name>LOGSEARCH_LOGFEEDER</name> + <timelineAppid>logfeeder</timelineAppid> + <displayName>Log Feeder</displayName> + <category>SLAVE</category> + <cardinality>ALL</cardinality> + <commandScript> + <script>scripts/logfeeder.py</script> + <scriptType>PYTHON</scriptType> + <timeout>5000</timeout> + </commandScript> + + <customCommands> + <customCommand> + <name>UPDATE_LOGFEEDER</name> + <commandScript> + <script>scripts/logfeeder.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </customCommand> + </customCommands> + + </component> + + </components> + + <osSpecifics> + <osSpecific> + <osFamily>redhat6</osFamily> + <packages> + <package><name>wget</name></package> + <package><name>tar</name></package> + <package><name>zip</name></package> + </packages> + </osSpecific> + </osSpecifics> + <configuration-dependencies> + <config-type>logsearch-config</config-type> + <config-type>solr-config</config-type> + <config-type>logfeeder-config</config-type> + </configuration-dependencies> + <restartRequiredAfterChange>false</restartRequiredAfterChange> + <widgetsFileName>widgets.json</widgetsFileName> + <metricsFileName>metrics.json</metricsFileName> + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/cb25daa3/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py new file mode 100644 index 0000000..8da6b82 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py @@ -0,0 +1,155 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys, os, pwd, grp, signal, time +from resource_management import * +from subprocess import call + +class Master(Script): + + #Call setup.sh to install the service + def install(self, env): + + #import properties defined in -config.xml file from params class + import params + import status_params + + # Install packages listed in metainfo.xml + self.install_packages(env) + + try: grp.getgrnam(params.logfeeder_group) + except KeyError: Group(group_name=params.logfeeder_group) + + try: pwd.getpwnam(params.logfeeder_user) + except KeyError: User(username=params.logfeeder_user, + gid=params.logfeeder_group, + groups=[params.logfeeder_group], + ignore_failures=True) + + Directory([params.logfeeder_log_dir, status_params.logfeeder_pid_dir, params.logfeeder_dir], + mode=0755, + cd_access='a', + owner=params.logfeeder_user, + group=params.logfeeder_group, + create_parents=True + ) + + + File(params.logfeeder_log, + mode=0644, + owner=params.logfeeder_user, + group=params.logfeeder_group, + content='' + ) + + + self.install_logfeeder() + Execute ('echo "logfeeder install complete"') + + + def install_logfeeder(self): + import params + if params.logfeeder_downloadlocation == 'RPM': + Execute('rpm -ivh http://TBD.rpm') + else: + Execute('cd ' + params.logfeeder_dir + '; wget ' + params.logfeeder_downloadlocation + ' -O logfeeder.tar.gz -a ' + params.logfeeder_log, user=params.logfeeder_user) + Execute('cd ' + params.logfeeder_dir + '; tar -xvf logfeeder.tar.gz', user=params.logfeeder_user) + + + def configure(self, env): + import params + env.set_params(params) + + #write content in jinja text field to system.properties + env_content=InlineTemplate(params.logfeeder_env_content) + File(format("{params.logfeeder_dir}/classes/logfeeder.properties"), content=env_content, owner=params.logfeeder_user) + + #update the log4j + file_content=InlineTemplate(params.logfeeder_log4j_content) + File(format("{params.logfeeder_dir}/classes/log4j.xml"), content=file_content, owner=params.logfeeder_user) + + config_content=InlineTemplate(params.logfeeder_config_content) + File(format("{params.logfeeder_dir}/classes/config.json"), content=config_content, owner=params.logfeeder_user) + + + #Call start.sh to start the service + def start(self, env): + + #import properties defined in -config.xml file from params class + import params + + #import status properties defined in -env.xml file from status_params class + import status_params + self.configure(env) + + #create pid/log dirs in case not there + Directory([params.logfeeder_log_dir, status_params.logfeeder_pid_dir], + mode=0755, + cd_access='a', + owner=params.logfeeder_user, + group=params.logfeeder_group, + create_parents=True + ) + + Execute('find '+params.service_packagedir+' -iname "*.sh" | xargs chmod +x') + cmd = params.service_packagedir + '/scripts/start_logfeeder.sh ' + params.logfeeder_dir + ' ' + params.logfeeder_log + ' ' + status_params.logfeeder_pid_file + ' ' + params.java64_home + ' ' + '-Xmx' + params.logfeeder_max_mem + + Execute('echo "Running cmd: ' + cmd + '"') + Execute(cmd, user=params.logfeeder_user) + + #Called to stop the service using the pidfile + def stop(self, env): + import params + + #import status properties defined in -env.xml file from status_params class + import status_params + + #this allows us to access the status_params.logfeeder_pid_file property as format('{logfeeder_pid_file}') + env.set_params(status_params) + + if os.path.isfile(status_params.logfeeder_pid_file): + Execute (format('kill `cat {logfeeder_pid_file}` >/dev/null 2>&1'), ignore_failures=True) + + #delete the pid file + Execute (format("rm -f {logfeeder_pid_file}"), user=params.logfeeder_user) + + #Called to get status of the service using the pidfile + def status(self, env): + + #import status properties defined in -env.xml file from status_params class + import status_params + env.set_params(status_params) + + #use built-in method to check status using pidfile + check_process_status(status_params.logfeeder_pid_file) + + def update_logfeeder(self, env): + import params + env.set_params(params) + Execute('echo Stopping logfeeder') + self.stop(env) + Execute('echo Updating logfeeder using latest install bits') + Execute(format("rm -rf {logfeeder_dir}/*")) + self.install_logfeeder() + Execute('echo Starting logfeeder') + self.start(env) + + +if __name__ == "__main__": + Master().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/cb25daa3/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py new file mode 100644 index 0000000..594d85f --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py @@ -0,0 +1,191 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys, os, pwd, grp, signal, time +from resource_management import * +from subprocess import call + +class Master(Script): + + #Call setup.sh to install the service + def install(self, env): + + #import properties defined in -config.xml file from params class + import params + import status_params + + # Install packages listed in metainfo.xml + self.install_packages(env) + + try: grp.getgrnam(params.logsearch_group) + except KeyError: Group(group_name=params.logsearch_group) + + try: pwd.getpwnam(params.logsearch_user) + except KeyError: User(username=params.logsearch_user, + gid=params.logsearch_group, + groups=[params.logsearch_group], + ignore_failures=True) + + Directory([params.logsearch_log_dir, status_params.logsearch_pid_dir, params.logsearch_dir], + mode=0755, + cd_access='a', + owner=params.logsearch_user, + group=params.logsearch_group, + create_parents=True + ) + + + File(params.logsearch_log, + mode=0644, + owner=params.logsearch_user, + group=params.logsearch_group, + content='' + ) + + self.install_logsearch() + + Execute ('echo "Logsearch install complete"') + + + def install_logsearch(self): + import params + if params.logsearch_downloadlocation == 'RPM': + Execute('rpm -ivh http://s3.amazonaws.com/dev2.hortonworks.com/ashishujjain/logsearch/logsearch_2_3_2_0_2950-0.0.1.2.3.2.0-2950.el6.x86_64.rpm') + else: + Execute('cd ' + params.logsearch_dir + '; wget ' + params.logsearch_downloadlocation + ' -O logsearch-portal.tar.gz -a ' + params.logsearch_log, user=params.logsearch_user) + Execute('cd ' + params.logsearch_dir + '; tar -xvf logsearch-portal.tar.gz', user=params.logsearch_user) + + + def configure(self, env): + import params + env.set_params(params) + + #write content in jinja text field to system.properties + env_content=InlineTemplate(params.logsearch_env_content) + File(format("{params.logsearch_dir}/classes/system.properties"), content=env_content, owner=params.logsearch_user) + + #update the log4j + file_content=InlineTemplate(params.logsearch_app_log4j_content) + File(format("{params.logsearch_dir}/classes/log4j.xml"), content=file_content, owner=params.logsearch_user) + + #write content in jinja text field to solrconfig.xml for service logs + file_content=InlineTemplate(params.logsearch_service_logs_solrconfig_content) + File(format("{params.logsearch_dir}/solr_configsets/hadoop_logs/conf/solrconfig.xml"), content=file_content, owner=params.logsearch_user) + + #write content in jinja text field to solrconfig.xml for audit logs + file_content=InlineTemplate(params.logsearch_audit_logs_solrconfig_content) + File(format("{params.logsearch_dir}/solr_configsets/audit_logs/conf/solrconfig.xml"), content=file_content, owner=params.logsearch_user) + + #Call start.sh to start the service + def start(self, env): + + #import properties defined in -config.xml file from params class + import params + + #import status properties defined in -env.xml file from status_params class + import status_params + self.configure(env) + + env.set_params(params) + Execute('echo metrics_collector_hosts ' + params.metrics_collector_hosts) + Execute('echo ambari_server_log_dir '+params.ambari_server_log_dir+' >> ' + params.logsearch_log, user=params.logsearch_user) + Execute('echo ambari_agent_log_dir '+params.ambari_agent_log_dir+' >> ' + params.logsearch_log, user=params.logsearch_user) + Execute('echo knox_log_dir '+params.knox_log_dir+' >> ' + params.logsearch_log, user=params.logsearch_user) + Execute('echo metrics_collector_log_dir '+params.metrics_collector_log_dir+' >> ' + params.logsearch_log, user=params.logsearch_user) + Execute('echo metrics_monitor_log_dir '+params.metrics_monitor_log_dir+' >> ' + params.logsearch_log, user=params.logsearch_user) + Execute('echo atlas_log_dir '+params.atlas_log_dir+' >> ' + params.logsearch_log, user=params.logsearch_user) + Execute('echo accumulo_log_dir '+params.accumulo_log_dir+' >> ' + params.logsearch_log, user=params.logsearch_user) + Execute('echo falcon_log_dir '+params.falcon_log_dir+' >> ' + params.logsearch_log, user=params.logsearch_user) + Execute('echo hbase_log_dir '+params.hbase_log_dir+' >> ' + params.logsearch_log, user=params.logsearch_user) + Execute('echo hdfs_log_dir_prefix '+params.hdfs_log_dir_prefix+' >> ' + params.logsearch_log, user=params.logsearch_user) + Execute('echo hive_log_dir '+params.hive_log_dir+' >> ' + params.logsearch_log, user=params.logsearch_user) + Execute('echo kafka_log_dir '+params.kafka_log_dir+' >> ' + params.logsearch_log, user=params.logsearch_user) + Execute('echo oozie_log_dir '+params.oozie_log_dir+' >> ' + params.logsearch_log, user=params.logsearch_user) + Execute('echo ranger_usersync_log_dir '+params.ranger_usersync_log_dir+' >> ' + params.logsearch_log, user=params.logsearch_user) + Execute('echo ranger_admin_log_dir '+params.ranger_admin_log_dir+' >> ' + params.logsearch_log, user=params.logsearch_user) + Execute('echo ranger_kms_log_dir '+params.ranger_kms_log_dir+' >> ' + params.logsearch_log, user=params.logsearch_user) + Execute('echo storm_log_dir '+params.storm_log_dir+' >> ' + params.logsearch_log, user=params.logsearch_user) + Execute('echo yarn_log_dir_prefix '+params.yarn_log_dir_prefix+' >> ' + params.logsearch_log, user=params.logsearch_user) + Execute('echo mapred_log_dir_prefix '+params.mapred_log_dir_prefix+' >> ' + params.logsearch_log, user=params.logsearch_user) + Execute('echo zk_log_dir '+params.zk_log_dir+' >> ' + params.logsearch_log, user=params.logsearch_user) + + + #create prerequisite Solr collections, if not already exist + #cmd = params.solr_bindir+'solr create -c '+params.logsearch_collection_service_logs+' -d '+params.logsearch_dir+'/solr_configsets/hadoop_logs/conf -s '+params.logsearch_numshards+' -rf ' + params.logsearch_repfactor + cmd = format('SOLR_INCLUDE={logsearch_solr_conf}/solr.in.sh {solr_bindir}/solr create -c {solr_collection_service_logs} -d {logsearch_dir}/solr_configsets/hadoop_logs/conf -s {logsearch_numshards} -rf {logsearch_repfactor}') + Execute('echo ' + cmd) + Execute(cmd, ignore_failures=True) + + #cmd = params.solr_bindir+'solr create -c history -d '+params.logsearch_dir+'/solr_configsets/history/conf -s '+params.logsearch_numshards+' -rf ' + params.logsearch_repfactor + cmd = format('SOLR_INCLUDE={logsearch_solr_conf}/solr.in.sh {solr_bindir}/solr create -c history -d {logsearch_dir}/solr_configsets/history/conf -s {logsearch_numshards} -rf {logsearch_repfactor}') + Execute('echo ' + cmd) + Execute(cmd, ignore_failures=True) + + if not(params.solr_audit_logs_use_ranger): + cmd = format('SOLR_INCLUDE={logsearch_solr_conf}/solr.in.sh {solr_bindir}/solr create -c {solr_collection_audit_logs} -d {logsearch_dir}/solr_configsets/audit_logs/conf -s {logsearch_numshards} -rf {logsearch_repfactor}') + Execute('echo ' + cmd) + Execute(cmd, ignore_failures=True) + + Execute('chmod -R ugo+r ' + params.logsearch_dir + '/solr_configsets') + + Execute('find '+params.service_packagedir+' -iname "*.sh" | xargs chmod +x') + cmd = params.service_packagedir + '/scripts/start_logsearch.sh ' + params.logsearch_dir + ' ' + params.logsearch_log + ' ' + status_params.logsearch_pid_file + ' ' + params.java64_home + ' ' + '-Xmx' + params.logsearch_app_max_mem + + Execute('echo "Running cmd: ' + cmd + '"') + Execute(cmd, user=params.logsearch_user) + + #Called to stop the service using the pidfile + def stop(self, env): + import params + + #import status properties defined in -env.xml file from status_params class + import status_params + + #this allows us to access the status_params.logsearch_pid_file property as format('{logsearch_pid_file}') + env.set_params(status_params) + + if os.path.isfile(status_params.logsearch_pid_file): + Execute (format('kill `cat {logsearch_pid_file}` >/dev/null 2>&1'), ignore_failures=True) + + #delete the pid file. Let's not, so startup can check and kill -9 if needed + #Execute (format("rm -f {logsearch_pid_file}"), user=params.logsearch_user) + + #Called to get status of the service using the pidfile + def status(self, env): + + #import status properties defined in -env.xml file from status_params class + import status_params + env.set_params(status_params) + + #use built-in method to check status using pidfile + check_process_status(status_params.logsearch_pid_file) + + def update_logsearch(self, env): + import params + env.set_params(params) + Execute('echo Stopping logsearch') + self.stop(env) + Execute('echo Updating logsearch using latest install bits') + Execute(format("rm -rf {logsearch_dir}/*")) + self.install_logsearch() + Execute('echo Starting logsearch') + self.start(env) + +if __name__ == "__main__": + Master().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/cb25daa3/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py new file mode 100644 index 0000000..3289708 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python + +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +import os +from resource_management.libraries.functions.default import default + +def get_port_from_url(address): + if not is_empty(address): + return address.split(':')[-1] + else: + return address + +# config object that holds the configurations declared in the -config.xml file +config = Script.get_config() + +hdp_version = default("/commandParams/version", None) + +#e.g. /var/lib/ambari-agent/cache/stacks/HDP/2.3/services/LOGSEARCH/package +service_packagedir = os.path.realpath(__file__).split('/scripts')[0] + +#shared configs +java64_home = config['hostLevelParams']['java_home'] +#get comma separated list of zookeeper hosts from clusterHostInfo +zookeeper_hosts = ",".join(config['clusterHostInfo']['zookeeper_hosts']) +cluster_name=str(config['clusterName']) + + +#TODO: pass all collectors +#if 'metrics_collector_hosts' in config['clusterHostInfo']: +# metrics_collector_hosts = ",".join(config['clusterHostInfo']['metrics_collector_hosts']) +#else: +# metrics_collector_hosts = '' + +#for now just pick first collector +if 'metrics_collector_hosts' in config['clusterHostInfo']: + metrics_collector_hosts = str(config['clusterHostInfo']['metrics_collector_hosts'][0]) + metrics_collector_port = str(get_port_from_url(config['configurations']['ams-site']['timeline.metrics.service.webapp.address'])) +else: + metrics_collector_hosts = '' + metrics_collector_port = '' + +#################################### +#Smart Config +#################################### +smart_solr_memory = config['configurations']['alpha-smart-config']['solr_memory'] +smart_solr_datadir = config['configurations']['alpha-smart-config']['solr_datadir'] + +##################################### +#Solr configs +##################################### + +# Only supporting HDPsearch and SolrCloud mode - so hardcode those options +solr_cloudmode='true' +solr_downloadlocation='HDPSEARCH' +#solr_cloudmode = config['configurations']['solr-config']['solr.cloudmode'] +#solr_downloadlocation = config['configurations']['solr-config']['solr.download.location'] +#solr_dir = config['configurations']['solr-config']['solr.dir'] + +solr_znode = config['configurations']['solr-config']['solr.znode'] +solr_port = config['configurations']['solr-env']['solr.port'] +solr_min_mem = format(config['configurations']['solr-config']['solr.minmem']) +solr_max_mem = format(config['configurations']['solr-config']['solr.maxmem']) +solr_instance_count = len(config['clusterHostInfo']['zookeeper_hosts']) +logsearch_solr_conf = config['configurations']['solr-config']['logsearch.solr.conf'] +logsearch_solr_datadir = format(config['configurations']['solr-config']['logsearch.solr.datadir']) +logsearch_solr_data_resources_dir = os.path.join(logsearch_solr_datadir,'resources') +logsearch_service_logs_max_retention = config['configurations']['logsearch-config']['logsearch_service_logs_max_retention'] +logsearch_audit_logs_max_retention = config['configurations']['logsearch-config']['logsearch_audit_logs_max_retention'] +logsearch_app_max_mem = config['configurations']['logsearch-config']['logsearch_app_max_mem'] + +zookeeper_port=default('/configurations/zoo.cfg/clientPort', None) +#get comma separated list of zookeeper hosts from clusterHostInfo +index = 0 +zookeeper_quorum="" +for host in config['clusterHostInfo']['zookeeper_hosts']: + zookeeper_quorum += host + ":"+str(zookeeper_port) + index += 1 + if index < len(config['clusterHostInfo']['zookeeper_hosts']): + zookeeper_quorum += "," + +if solr_downloadlocation == 'HDPSEARCH': + solr_dir='/opt/lucidworks-hdpsearch/solr' + solr_bindir = solr_dir + '/bin' + cloud_scripts=solr_dir+'/server/scripts/cloud-scripts' +else: + solr_bindir = solr_dir + '/latest/bin' + cloud_scripts=solr_dir+'/latest/server/scripts/cloud-scripts' + + +solr_user = config['configurations']['solr-env']['solr.user'] +solr_group = config['configurations']['solr-env']['solr.group'] +solr_log_dir = config['configurations']['solr-env']['solr.log.dir'] +solr_log = solr_log_dir+'/solr-install.log' + +solr_piddir = config['configurations']['solr-env']['solr_pid_dir'] +solr_pidfile = format("{solr_piddir}/solr-{solr_port}.pid") + +solr_env_content = config['configurations']['solr-env']['content'] + +solr_xml_content = config['configurations']['solr-xml-env']['content'] + +solr_log4j_content = config['configurations']['solr-log4j-env']['content'] + +solr_zoo_content = config['configurations']['solr-zoo-env']['content'] + +solr_sh_content = config['configurations']['solr-sh']['content'] + + +##################################### +#Logsearch configs +##################################### + +logsearch_downloadlocation = config['configurations']['logsearch-env']['logsearch_download_location'] +if logsearch_downloadlocation == 'RPM': + logsearch_dir = '/usr/hdp/'+hdp_version+'/logsearch' +else: + logsearch_dir = config['configurations']['logsearch-env']['logsearch_dir'] + + + +logsearch_downloadlocation = config['configurations']['logsearch-env']['logsearch_download_location'] +logsearch_collection_service_logs = default('/configurations/logsearch-config/logsearch_collection_service_logs', 'hadoop_logs') +logsearch_collection_audit_logs = default('/configurations/logsearch-config/logsearch_collection_audit_logs', 'audit_logs') +#logsearch_numshards = str(config['configurations']['logsearch-config']['logsearch_collection_numshards']) +logsearch_numshards = format(str(solr_instance_count)) +logsearch_repfactor = str(config['configurations']['logsearch-config']['logsearch_collection_rep_factor']) + +solr_collection_service_logs = default('/configurations/logsearch-config/solr_collection_service_logs', 'hadoop_logs') +solr_collection_audit_logs = default('/configurations/logsearch-config/solr_collection_audit_logs', 'audit_logs') + +solr_audit_logs_use_ranger = default('/configurations/logsearch-config/solr_audit_logs_use_ranger', 'false') +solr_audit_logs_url = '' + +if solr_audit_logs_use_ranger: + #In Ranger, this contain the /zkNode also + ranger_audit_solr_zookeepers = default('/configurations/ranger-admin-site/ranger.audit.solr.zookeepers', None) + #TODO: ranger property already has zk node appended. We need to remove it. + #For now, let's assume it is going to be URL + solr_audit_logs_url = default('/configurations/ranger-admin-site/ranger.audit.solr.urls', solr_audit_logs_url) +else: + solr_audit_logs_zk_quorum = default('/configurations/logsearch-config/solr_audit_logs_zk_quorum', None) + solr_audit_logs_zk_node = default('/configurations/logsearch-config/solr_audit_logs_zk_node', None) + + solr_audit_logs_zk_node = format(solr_audit_logs_zk_node) + solr_audit_logs_zk_quorum = format(solr_audit_logs_zk_quorum) + + if not(solr_audit_logs_zk_quorum): + solr_audit_logs_zk_quorum=zookeeper_quorum + if not(solr_audit_logs_zk_node): + solr_audit_logs_zk_node=solr_znode + +#using zookeeper znode for solr instead of host/port now that logsearch system.properties supports it +#solr_host = config['configurations']['logsearch-config']['solr_host'] +#solr_port = str(config['configurations']['logsearch-config']['solr_port']) + + +# logsearch-env configs +logsearch_user = config['configurations']['logsearch-env']['logsearch_user'] +logsearch_group = config['configurations']['logsearch-env']['logsearch_group'] +logsearch_log_dir = config['configurations']['logsearch-env']['logsearch_log_dir'] +logsearch_log = logsearch_log_dir+'/logsearch.out' + +# store the log file for the service from the 'solr.log' property of the 'logsearch-env.xml' file +logsearch_env_content = config['configurations']['logsearch-env']['content'] +logsearch_service_logs_solrconfig_content = config['configurations']['logsearch-service_logs-solrconfig']['content'] +logsearch_audit_logs_solrconfig_content = config['configurations']['logsearch-audit_logs-solrconfig']['content'] +logsearch_app_log4j_content = config['configurations']['logsearch-app-log4j']['content'] + +#Log dirs +ambari_server_log_dir = '/var/log/ambari-server' +ambari_agent_log_dir = '/var/log/ambari-agent' +knox_log_dir = '/var/log/knox' + +metrics_collector_log_dir = default('/configurations/ams-env/metrics_collector_log_dir', '/var/log') +metrics_monitor_log_dir = default('/configurations/ams-env/metrics_monitor_log_dir', '/var/log') + + +atlas_log_dir = default('/configurations/atlas-env/metadata_log_dir', '/var/log/atlas') +accumulo_log_dir = default('/configurations/accumulo-env/accumulo_log_dir', '/var/log/accumulo') +falcon_log_dir = default('/configurations/falcon-env/falcon_log_dir','/var/log/falcon') +hbase_log_dir = default('/configurations/hbase-env/hbase_log_dir','/var/log/hbase') +hdfs_log_dir_prefix = default('/configurations/hadoop-env/hdfs_log_dir_prefix','/var/log/hadoop') +hive_log_dir = default('/configurations/hive-env/hive_log_dir','/var/log/hive') +kafka_log_dir = default('/configurations/kafka-env/kafka_log_dir','/var/log/kafka') +oozie_log_dir = default('/configurations/oozie-env/oozie_log_dir','/var/log/oozie') +ranger_usersync_log_dir = default('/configurations/ranger-env/ranger_usersync_log_dir', '/var/log/ranger/usersync') +ranger_admin_log_dir = default('/configurations/ranger-env/ranger_admin_log_dir', '/var/log/ranger/admin') +ranger_kms_log_dir = default('/configurations/kms-env/kms_log_dir', '/var/log/ranger/kms') +storm_log_dir = default('/configurations/storm-env/storm_log_dir','/var/log/storm') +yarn_log_dir_prefix = default('/configurations/yarn-env/yarn_log_dir_prefix','/var/log/hadoop') +mapred_log_dir_prefix = default('/configurations/mapred-env/mapred_log_dir_prefix','/var/log/hadoop') +zk_log_dir = default('/configurations/zookeeper-env/zk_log_dir','/var/log/zookeeper') + + +##################################### +#Logfeeder configs +##################################### + +logfeeder_downloadlocation = config['configurations']['logfeeder-env']['logfeeder_download_location'] +if logfeeder_downloadlocation == 'RPM': + logfeeder_dir = '/usr/hdp/'+hdp_version+'/logfeeder' +else: + logfeeder_dir = config['configurations']['logfeeder-env']['logfeeder_dir'] + +logfeeder_downloadlocation = config['configurations']['logfeeder-env']['logfeeder_download_location'] + + + + +# logfeeder-config configs + +solr_service_logs_enable = default('/configurations/logfeeder-config/solr_service_logs_enable',True) +solr_audit_logs_enable = default('/configurations/logfeeder-config/solr_audit_logs_enable',True) + +kafka_broker_list = default('/configurations/logfeeder-config/kafka_broker_list', '') +kafka_security_protocol= default('/configurations/logfeeder-config/kafka_security_protocol', '') +kafka_kerberos_service_name=default('/configurations/logfeeder-config/kafka_kerberos_service_name', '') + +kafka_service_logs_enable = default('/configurations/logfeeder-config/kafka_service_logs_enable',False) +kafka_audit_logs_enable = default('/configurations/logfeeder-config/kafka_audit_logs_enable',False) +kafka_topic_service_logs = default('/configurations/logfeeder-config/kafka_topic_service_logs','service_logs') +kafka_topic_audit_logs = default('/configurations/logfeeder-config/kafka_topic_audit_logs','audit_logs') + + +# logfeeder-env configs +logfeeder_user = config['configurations']['logfeeder-env']['logfeeder_user'] +logfeeder_group = config['configurations']['logfeeder-env']['logfeeder_group'] +logfeeder_log_dir = config['configurations']['logfeeder-env']['logfeeder_log_dir'] +logfeeder_log = logfeeder_log_dir+'/logfeeder.out' +logfeeder_max_mem = config['configurations']['logfeeder-env']['logfeeder_max_mem'] +logfeeder_env_content = config['configurations']['logfeeder-env']['content'] +logfeeder_config_content = config['configurations']['logfeeder-input-configs']['content'] +logfeeder_log4j_content = config['configurations']['logfeeder-log4j']['content'] +logfeeder_pid_dir = config['configurations']['logfeeder-env']['logfeeder_pid_dir'] http://git-wip-us.apache.org/repos/asf/ambari/blob/cb25daa3/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_solr.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_solr.sh b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_solr.sh new file mode 100755 index 0000000..b672e1c --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_solr.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + + +#Path to install Solr to e.g. /opt/solr +SOLR_PATH=$1 + +#solr user e.g. solr +SOLR_USER=$2 + + +echo "Starting Solr install" + +getent passwd $SOLR_USER +if [ $? -eq 0 ]; then + echo "the user exists, no need to create" +else + echo "creating solr user" + adduser $SOLR_USER +fi + + +#hadoop fs -test -d /user/$SOLR_USER +#if [ $? -eq 1 ]; then +# echo "Creating user dir in HDFS" +# sudo -u hdfs hdfs dfs -mkdir -p /user/$SOLR_USER +# sudo -u hdfs hdfs dfs -chown $SOLR_USER /user/solr +#fi + + http://git-wip-us.apache.org/repos/asf/ambari/blob/cb25daa3/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/solr.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/solr.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/solr.py new file mode 100644 index 0000000..cdbfabd --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/solr.py @@ -0,0 +1,177 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys, os, pwd, grp, signal, time +from resource_management import * +from subprocess import call + +class Master(Script): + + #Call setup_solr.sh to install the service + def install(self, env): + + #import properties defined in -config.xml file from params class + import params + import status_params + + # Install packages listed in metainfo.xml + self.install_packages(env) + + Execute('find '+params.service_packagedir+' -iname "*.sh" | xargs chmod +x') + + try: grp.getgrnam(params.solr_group) + except KeyError: Group(group_name=params.solr_group) + + try: pwd.getpwnam(params.solr_user) + except KeyError: User(username=params.solr_user, + gid=params.solr_group, + groups=[params.solr_group], + ignore_failures=True) + + Directory([params.solr_log_dir, status_params.solr_piddir, params.solr_dir, params.logsearch_solr_conf, params.logsearch_solr_datadir, params.logsearch_solr_data_resources_dir], + mode=0755, + cd_access='a', + owner=params.solr_user, + group=params.solr_group, + create_parents=True + ) + + + File(params.solr_log, + mode=0644, + owner=params.solr_user, + group=params.solr_group, + content='' + ) + + Execute('echo Solr dir: ' + params.solr_dir) + + if params.solr_bindir == 'UNDEFINED' or params.cloud_scripts == 'UNDEFINED': + Execute('echo Error: solr_bin: ' + params.solr_bindir + ' cloud_scripts: ' + params.cloud_scripts) + + + if params.solr_downloadlocation == 'HDPSEARCH': + Execute('export JAVA_HOME='+params.java64_home+';yum install -y lucidworks-hdpsearch') + + #form command to invoke setup_solr.sh with its arguments and execute it + cmd = params.service_packagedir + '/scripts/setup_solr.sh ' + params.solr_dir + ' ' + params.solr_user + ' >> ' + params.solr_log + ' 2>&1' + Execute('echo "Running ' + cmd + '" as root') + Execute(cmd, ignore_failures=True) + + if params.solr_downloadlocation == 'HDPSEARCH': + Execute('echo HDPSeach mode selected') + else: + Execute('cd ' + params.solr_dir + '; wget ' + params.solr_downloadlocation + ' -O solr.tgz -a ' + params.solr_log, user=params.solr_user) + Execute('cd ' + params.solr_dir + '; tar -xvf solr.tgz', user=params.solr_user) + Execute('cd ' + params.solr_dir + '; ln -s solr-* latest', user=params.solr_user) + + #ensure all solr files owned by solr + Execute('chown -R '+params.solr_user + ':' + params.solr_group + ' ' + params.solr_dir) + + Execute ('echo "Solr install complete"') + + + + def configure(self, env): + import params + env.set_params(params) + + #write content in jinja text field to solr.in.sh + env_content=InlineTemplate(params.solr_env_content) + File(format("{logsearch_solr_conf}/solr.in.sh"), content=env_content, owner=params.solr_user) + #File(format("{params.solr_bindir}/solr.in.sh"), content=env_content, owner=params.solr_user) + + xml_content=InlineTemplate(params.solr_xml_content) + File(format("{logsearch_solr_datadir}/solr.xml"), content=xml_content, owner=params.solr_user) + + log4j_content=InlineTemplate(params.solr_log4j_content) + File(format("{logsearch_solr_datadir}/resources/log4j.properties"), content=log4j_content, owner=params.solr_user) + + zoo_content=InlineTemplate(params.solr_zoo_content) + File(format("{logsearch_solr_datadir}/zoo.cfg"), content=zoo_content, owner=params.solr_user) + + solr_sh_content=InlineTemplate(params.solr_sh_content) + File(format("{params.solr_bindir}/logsearch_solr.sh"), mode=0755, content=solr_sh_content, owner=params.solr_user) + + + #Call start.sh to start the service + def start(self, env): + + #import properties defined in -config.xml file from params class + import params + + #import status properties defined in -env.xml file from status_params class + import status_params + self.configure(env) + + #this allows us to access the params.solr_pidfile property as format('{solr_pidfile}') + env.set_params(params) + + Execute('find '+params.service_packagedir+' -iname "*.sh" | xargs chmod +x') + + + #form command to invoke start.sh with its arguments and execute it + if params.solr_cloudmode: + Execute ('echo "Creating znode" ' + params.solr_znode) + Execute ('echo "' + params.cloud_scripts + '/zkcli.sh -zkhost ' + params.zookeeper_hosts + ' -cmd makepath ' + params.solr_znode + '"') + Execute ('export JAVA_HOME='+params.java64_home+';'+params.cloud_scripts + '/zkcli.sh -zkhost ' + params.zookeeper_hosts + ' -cmd makepath ' + params.solr_znode, user=params.solr_user, ignore_failures=True ) + + #$SOLR_IN_PATH/solr.in.sh ./solr start -cloud -noprompt -s $SOLR_DATA_DIR + Execute(format('SOLR_INCLUDE={logsearch_solr_conf}/solr.in.sh {solr_bindir}/solr start -cloud -noprompt -s {logsearch_solr_datadir} >> {solr_log} 2>&1'), user=params.solr_user) + #Seems when logsearch comes up, solr is not ready. So let's give sometime for it to initialize + time.sleep(15) + + else: + cmd = params.service_packagedir + '/scripts/start.sh ' + params.solr_dir + ' ' + params.solr_log + ' ' + status_params.solr_pidfile + ' ' + params.solr_bindir + Execute('echo "Running cmd: ' + cmd + '"') + Execute(cmd, user=params.solr_user) + + #Called to stop the service using the pidfile + def stop(self, env): + import params + + #import status properties defined in -env.xml file from status_params class + import status_params + + #this allows us to access the params.solr_pidfile property as format('{solr_pidfile}') + env.set_params(params) + #self.configure(env) + + if os.path.isfile(status_params.solr_pidfile): + + #kill the instances of solr + Execute (format('SOLR_INCLUDE={logsearch_solr_conf}/solr.in.sh {solr_bindir}/solr stop -all >> {solr_log}'), user=params.solr_user) + + #delete the pid file + #Execute (format("rm -f {solr_pidfile} >> {solr_log}"), user=params.solr_user) + + #Called to get status of the service using the pidfile + def status(self, env): + + #import status properties defined in -env.xml file from status_params class + import status_params + env.set_params(status_params) + + #use built-in method to check status using pidfile + check_process_status(status_params.solr_pidfile) + + + +if __name__ == "__main__": + Master().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/cb25daa3/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/start_logfeeder.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/start_logfeeder.sh b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/start_logfeeder.sh new file mode 100755 index 0000000..fc1e54b --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/start_logfeeder.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -e +if [ $# -lt 4 ]; then + echo "Error: Not enough parameters. Count=$# excpected 4 or more" + echo "Usage: <logfeeder path> <logfile> <pid file> <java_home> [java_mem]" + echo "Example: $0 /opt/logfeeder /var/log/logfeeder/logfeeder.log /var/run/logfeeder/logfeeder.pid /usr/jdk64/jdk1.8.0_45 -Xmx512m" + exit 1 +fi + +#!/bin/bash +set -e + +#path containing start.jar file e.g. /opt/solr/latest/server +export LOGFEEDER_PATH=$1 + +#Logfile e.g. /var/log/solr.log +export LOGFILE=$2 + +#pid file e.g. /var/run/solr.pid +export PID_FILE=$3 + +export JAVA_HOME=$4 + +export LOGFEEDER_JAVA_MEM=$5 +if [ "$LOGFEEDER_JAVA_MEM" = "" ]; then + export LOGFEEDER_JAVA_MEM="-Xmx512m" +fi + +cd $LOGFEEDER_PATH +echo "Starting Logfeeder logfile=$LOGFILE, PID_FILE=$PID_FILE, JAVA_HOME=$JAVA_HOME, MEM=$LOGFEEDER_JAVA_MEM ..." +./run.sh http://git-wip-us.apache.org/repos/asf/ambari/blob/cb25daa3/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/start_logsearch.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/start_logsearch.sh b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/start_logsearch.sh new file mode 100755 index 0000000..795baf6 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/start_logsearch.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -e + +#path containing start.jar file e.g. /opt/solr/latest/server +export LOGSEARCH_PATH=$1 + +#Logfile e.g. /var/log/solr.log +export LOGFILE=$2 + +#pid file e.g. /var/run/solr.pid +export PID_FILE=$3 + +export JAVA_HOME=$4 + +export LOGSEARCH_JAVA_MEM=$5 +if [ "$LOGSEARCH_JAVA_MEM" = "" ]; then + export LOGSEARCH_JAVA_MEM="-Xmx1g" +fi + +cd $LOGSEARCH_PATH +echo "Starting Logsearch..." +./run.sh http://git-wip-us.apache.org/repos/asf/ambari/blob/cb25daa3/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/status_params.py new file mode 100755 index 0000000..f9ed87c --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/status_params.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +import glob + +# config object that holds the status related configurations declared in the -env.xml file +config = Script.get_config() + +#solr pid file +solr_piddir = config['configurations']['solr-env']['solr_pid_dir'] +solr_port = config['configurations']['solr-env']['solr.port'] +solr_pidfile = format("{solr_piddir}/solr-{solr_port}.pid") + + +#logsearch pid file +logsearch_pid_dir = config['configurations']['logsearch-env']['logsearch_pid_dir'] +logsearch_pid_file = format("{logsearch_pid_dir}/logsearch.pid") + +#logfeeder pid file +logfeeder_pid_dir = config['configurations']['logfeeder-env']['logfeeder_pid_dir'] +logfeeder_pid_file = format("{logfeeder_pid_dir}/logfeeder.pid") http://git-wip-us.apache.org/repos/asf/ambari/blob/cb25daa3/ambari-server/src/main/resources/stacks/HDP/2.4/services/LOGSEARCH/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/services/LOGSEARCH/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/services/LOGSEARCH/metainfo.xml new file mode 100644 index 0000000..df697dc --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/services/LOGSEARCH/metainfo.xml @@ -0,0 +1,26 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>LOGSEARCH</name> + <extends>common-services/LOGSEARCH/0.5.0</extends> + </service> + </services> +</metainfo>
