AMBARI-22647. Rafactor: Package Log Search and Log Feeder into jars + cleanup start scripts (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/562795c4 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/562795c4 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/562795c4 Branch: refs/heads/branch-feature-AMBARI-22008-isilon Commit: 562795c4d35bf3b45d196856e9d66e5dfdd0a17b Parents: f9ad13d Author: Oliver Szabo <oleew...@gmail.com> Authored: Fri Dec 15 22:17:46 2017 +0100 Committer: Oliver Szabo <oleew...@gmail.com> Committed: Mon Dec 18 16:28:38 2017 +0100 ---------------------------------------------------------------------- .../ambari-logsearch-assembly/pom.xml | 117 +++----- .../src/main/package/deb/logfeeder/postinst | 22 ++ .../src/main/package/deb/logfeeder/postrm | 6 + .../src/main/package/deb/logfeeder/posttrm | 15 - .../src/main/package/deb/logfeeder/preinst | 13 + .../src/main/package/deb/portal/postinst | 19 ++ .../src/main/package/deb/portal/postrm | 6 + .../src/main/package/deb/portal/preinst | 13 + .../main/package/rpm/logfeeder/postinstall.sh | 37 +++ .../main/package/rpm/logfeeder/postremove.sh | 21 ++ .../main/package/rpm/logfeeder/preinstall.sh | 28 ++ .../src/main/package/rpm/portal/postinstall.sh | 34 +++ .../src/main/package/rpm/portal/postremove.sh | 21 ++ .../src/main/package/rpm/portal/preinstall.sh | 28 ++ .../ambari-logsearch-logfeeder/README.md | 4 +- .../ambari-logsearch-logfeeder/build.properties | 18 -- .../ambari-logsearch-logfeeder/build.xml | 28 +- .../ambari-logsearch-logfeeder/pom.xml | 31 -- .../ambari-logsearch-logfeeder/run.sh | 4 +- .../org/apache/ambari/logfeeder/LogFeeder.java | 35 +-- .../ambari/logfeeder/LogFeederCommandLine.java | 63 ++-- .../ambari/logfeeder/common/ConfigHandler.java | 7 +- .../src/main/package/deb/control/control | 22 -- .../src/main/package/deb/control/postinst | 15 - .../src/main/package/deb/control/postrm | 15 - .../src/main/package/deb/control/preinst | 15 - .../src/main/package/deb/control/prerm | 15 - .../src/main/scripts/logfeeder-env.sh | 40 +++ .../src/main/scripts/logfeeder.sh | 292 +++++++++++++++++++ .../src/main/scripts/run.sh | 103 ------- .../ambari-logsearch-server/README.md | 2 +- .../ambari-logsearch-server/build.xml | 36 ++- .../ambari-logsearch-server/pom.xml | 11 - ambari-logsearch/ambari-logsearch-server/run.sh | 2 +- .../org/apache/ambari/logsearch/LogSearch.java | 2 +- .../src/main/scripts/logsearch-env.sh | 40 +++ .../src/main/scripts/logsearch.sh | 274 +++++++++++++++++ .../src/main/scripts/run.sh | 99 ------- .../src/main/scripts/stop.sh | 44 --- ambari-logsearch/docker/Dockerfile | 6 +- ambari-logsearch/docker/all.yml | 4 +- ambari-logsearch/docker/bin/start.sh | 30 +- ambari-logsearch/docker/docker-compose.yml | 2 - ambari-logsearch/docker/logsearch-logfeeder.yml | 1 - ambari-logsearch/docker/logsearch-server.yml | 1 - .../test-config/logfeeder/logfeeder-env.sh | 5 +- .../test-config/logfeeder/logfeeder.properties | 2 +- .../test-config/logsearch/logsearch-env.sh | 5 +- .../logsearch/logsearch-https.properties | 4 +- .../test-config/logsearch/logsearch.properties | 4 +- .../0.5.0/configuration/logfeeder-env.xml | 2 +- .../configuration/logfeeder-properties.xml | 2 +- .../0.5.0/configuration/logsearch-env.xml | 6 +- .../0.5.0/package/scripts/logfeeder.py | 7 +- .../0.5.0/package/scripts/logsearch.py | 8 +- .../0.5.0/package/scripts/logsearch_common.py | 54 ---- .../LOGSEARCH/0.5.0/package/scripts/params.py | 8 +- .../0.5.0/package/scripts/setup_logfeeder.py | 2 +- .../0.5.0/package/scripts/setup_logsearch.py | 2 +- .../0.5.0/properties/logfeeder-env.sh.j2 | 5 +- .../0.5.0/properties/logsearch-env.sh.j2 | 5 +- .../stack-hooks/after-INSTALL/scripts/params.py | 2 +- .../hooks/after-INSTALL/test_after_install.py | 8 +- .../stacks/2.4/LOGSEARCH/test_logfeeder.py | 34 +-- .../stacks/2.4/LOGSEARCH/test_logsearch.py | 31 +- .../test/python/stacks/2.4/configs/default.json | 2 +- 66 files changed, 1102 insertions(+), 737 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-assembly/pom.xml b/ambari-logsearch/ambari-logsearch-assembly/pom.xml index cbc62ce..19c3d90 100644 --- a/ambari-logsearch/ambari-logsearch-assembly/pom.xml +++ b/ambari-logsearch/ambari-logsearch-assembly/pom.xml @@ -82,6 +82,18 @@ <configuration> <group>Development</group> <name>${logsearch.portal.package.name}</name> + <postinstallScriptlet> + <scriptFile>${project.basedir}/src/main/package/rpm/portal/postinstall.sh</scriptFile> + <fileEncoding>utf-8</fileEncoding> + </postinstallScriptlet> + <postremoveScriptlet> + <scriptFile>${project.basedir}/src/main/package/rpm/portal/postremove.sh</scriptFile> + <fileEncoding>utf-8</fileEncoding> + </postremoveScriptlet> + <preinstallScriptlet> + <scriptFile>${project.basedir}/src/main/package/rpm/portal/preinstall.sh</scriptFile> + <fileEncoding>utf-8</fileEncoding> + </preinstallScriptlet> <mappings> <mapping> <directory>${logsearch.portal.mapping.path}</directory> @@ -89,37 +101,11 @@ <source> <location>${logsearch.server.dir}/target/package</location> <excludes> - <exclude>classes/log4j.xml</exclude> - <exclude>classes/logsearch.properties</exclude> - <exclude>classes/user_pass.json</exclude> - <exclude>classes/HadoopServiceConfig.json</exclude> - <exclude>solr_configsets/**</exclude> <exclude>libs/checkstyle*.jar</exclude> </excludes> </source> </sources> </mapping> - <mapping> - <directory>${logsearch.portal.conf.mapping.path}</directory> - <sources> - <source> - <location>${logsearch.server.dir}/target/package/classes</location> - <includes> - <include>log4j.xml</include> - <include>logsearch.properties</include> - <include>user_pass.json</include> - </includes> - </source> - </sources> - </mapping> - <mapping> - <directory>${logsearch.portal.conf.mapping.path}/solr_configsets</directory> - <sources> - <source> - <location>${logsearch.server.dir}/target/package/solr_configsets</location> - </source> - </sources> - </mapping> </mappings> </configuration> </execution> @@ -132,6 +118,18 @@ <configuration> <group>Development</group> <name>${logsearch.logfeeder.package.name}</name> + <postinstallScriptlet> + <scriptFile>${project.basedir}/src/main/package/rpm/logfeeder/postinstall.sh</scriptFile> + <fileEncoding>utf-8</fileEncoding> + </postinstallScriptlet> + <postremoveScriptlet> + <scriptFile>${project.basedir}/src/main/package/rpm/logfeeder/postremove.sh</scriptFile> + <fileEncoding>utf-8</fileEncoding> + </postremoveScriptlet> + <preinstallScriptlet> + <scriptFile>${project.basedir}/src/main/package/rpm/logfeeder/preinstall.sh</scriptFile> + <fileEncoding>utf-8</fileEncoding> + </preinstallScriptlet> <mappings> <mapping> <directory>${logsearch.logfeeder.mapping.path}</directory> @@ -139,27 +137,11 @@ <source> <location>${logsearch.logfeeder.dir}/target/package</location> <excludes> - <exclude>classes/log4j.xml</exclude> - <exclude>classes/logfeeder.properties</exclude> - <exclude>classes/grok-patterns</exclude> <exclude>libs/checkstyle*.jar</exclude> </excludes> </source> </sources> </mapping> - <mapping> - <directory>${logsearch.logfeeder.conf.mapping.path}</directory> - <sources> - <source> - <location>${logsearch.logfeeder.dir}/target/package/classes</location> - <includes> - <include>log4j.xml</include> - <include>logfeeder.properties</include> - <include>grok-patterns</include> - </includes> - </source> - </sources> - </mapping> </mappings> </configuration> </execution> @@ -197,7 +179,11 @@ <directory>${project.basedir}/src/main/package/deb</directory> <excludes> <exclude>portal/postinst</exclude> + <exclude>portal/postrm</exclude> + <exclude>portal/preinstall</exclude> <exclude>logfeeder/postinst</exclude> + <exclude>logfeeder/postrm</exclude> + <exclude>logfeeder/preinstall</exclude> </excludes> <filtering>false</filtering> </resource> @@ -205,7 +191,11 @@ <directory>${project.basedir}/src/main/package/deb</directory> <includes> <include>portal/postinst</include> + <include>portal/postrm</include> + <include>portal/preinstall</include> <include>logfeeder/postinst</include> + <include>logfeeder/postrm</include> + <include>logfeeder/preinstall</include> </includes> <filtering>true</filtering> </resource> @@ -242,34 +232,9 @@ <group>root</group> </mapper> <excludes> - classes/log4j.xml,classes/logsearch.properties,classes/user_pass.json,classes/HadoopServiceConfig.json,solr_configsets/**,libs/checkstyle*.jar + libs/checkstyle*.jar </excludes> </data> - <data> - <src>${logsearch.server.dir}/target/package/classes</src> - <type>directory</type> - <mapper> - <prefix>${logsearch.portal.conf.mapping.path}</prefix> - <type>perm</type> - <user>root</user> - <group>root</group> - <filemode>644</filemode> - </mapper> - <includes> - log4j.xml,logsearch.properties,user_pass.json - </includes> - </data> - <data> - <src>${logsearch.server.dir}/target/package/solr_configsets</src> - <type>directory</type> - <mapper> - <prefix>${logsearch.portal.conf.mapping.path}/solr_configsets</prefix> - <type>perm</type> - <user>root</user> - <group>root</group> - </mapper> - <excludes>libs/checkstyle*.jar</excludes> - </data> </dataSet> </configuration> </execution> @@ -297,23 +262,9 @@ <group>root</group> </mapper> <excludes> - classes/log4j.xml,classes/logfeeder.properties,classes/grok-patterns,libs/checkstyle*.jar + libs/checkstyle*.jar </excludes> </data> - <data> - <src>${logsearch.logfeeder.dir}/target/package/classes</src> - <type>directory</type> - <mapper> - <prefix>${logsearch.logfeeder.conf.mapping.path}</prefix> - <type>perm</type> - <user>root</user> - <group>root</group> - <filemode>644</filemode> - </mapper> - <includes> - log4j.xml,logfeeder.properties,grok-patterns - </includes> - </data> </dataSet> </configuration> </execution> http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/postinst ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/postinst b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/postinst index 21a01fa..a4eec77 100644 --- a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/postinst +++ b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/postinst @@ -13,3 +13,25 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License + +LOGFEEDER_SCRIPT_LINK_NAME="/usr/bin/logfeeder" +LOGFEEDER_SCRIPT_SOURCE="/usr/lib/ambari-logsearch-logfeeder/bin/logfeeder.sh" + +LOGFEEDER_CONF_LINK="/etc/ambari-logsearch-logfeeder/conf" +LOGFEEDER_CONF_SOURCE="/usr/lib/ambari-logsearch-logfeeder/conf" + +ln -s $LOGFEEDER_SCRIPT_SOURCE $LOGFEEDER_SCRIPT_LINK_NAME +ln -s $LOGFEEDER_CONF_SOURCE $LOGFEEDER_CONF_LINK + +# handle old checkpoint & keys folder + +LOGFEEDER_CONF_BACKUP="/usr/lib/ambari-logsearch-logfeeder/conf-old" + +if [ -d "$LOGFEEDER_CONF_BACKUP" ]; then + if [ -d "$LOGFEEDER_CONF_BACKUP/keys" ]; then + cp -r $LOGFEEDER_CONF_BACKUP/keys $LOGFEEDER_CONF_SOURCE + fi + if [ -d "$LOGFEEDER_CONF_BACKUP/checkpoints" ]; then + cp -r $LOGFEEDER_CONF_BACKUP/checkpoints $LOGFEEDER_CONF_SOURCE + fi +fi \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/postrm ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/postrm b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/postrm index 21a01fa..5f1c623 100644 --- a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/postrm +++ b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/postrm @@ -13,3 +13,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License + +LOGFEEDER_SCRIPT_LINK_NAME="/usr/bin/logfeeder" +LOGFEEDER_CONF_DIR_LINK="/etc/ambari-logsearch-logfeeder/conf" + +rm -f $LOGFEEDER_SCRIPT_LINK_NAME +rm -f $LOGFEEDER_CONF_DIR_LINK \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/posttrm ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/posttrm b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/posttrm deleted file mode 100644 index 21a01fa..0000000 --- a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/posttrm +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/preinst ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/preinst b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/preinst index 21a01fa..01d70e0 100644 --- a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/preinst +++ b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/preinst @@ -13,3 +13,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License + +LOGFEEDER_CONF="/etc/ambari-logsearch-logfeeder/conf" +LOGFEEDER_CONF_BACKUP="/usr/lib/ambari-logsearch-logfeeder/conf-old" + +if [ -d $LOGFEEDER_CONF_BACKUP ]; then + rm -rf $LOGFEEDER_CONF_BACKUP +fi + +if [ -d $LOGFEEDER_CONF ]; then + mkdir -p $LOGFEEDER_CONF_BACKUP + cp -r $LOGFEEDER_CONF/* $LOGFEEDER_CONF_BACKUP + rm -rf $LOGFEEDER_CONF +fi \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/postinst ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/postinst b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/postinst index 21a01fa..9107482 100644 --- a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/postinst +++ b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/postinst @@ -13,3 +13,22 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License + +LOGSEARCH_SCRIPT_LINK_NAME="/usr/bin/logsearch" +LOGSEARCH_SCRIPT_SOURCE="/usr/lib/ambari-logsearch-portal/bin/logsearch.sh" + +LOGSEARCH_CONF_LINK="/etc/ambari-logsearch-portal/conf" +LOGSEARCH_CONF_SOURCE="/usr/lib/ambari-logsearch-portal/conf" + +ln -s $LOGSEARCH_SCRIPT_SOURCE $LOGSEARCH_SCRIPT_LINK_NAME +ln -s $LOGSEARCH_CONF_SOURCE $LOGSEARCH_CONF_LINK + +# handle old keys folder + +LOGSEARCH_CONF_BACKUP="/usr/lib/ambari-logsearch-portal/conf-old" + +if [ -d "$LOGSEARCH_CONF_BACKUP" ]; then + if [ -d "$LOGSEARCH_CONF_BACKUP/keys" ]; then + cp -r $LOGSEARCH_CONF_BACKUP/keys $LOGSEARCH_CONF_SOURCE + fi +fi \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/postrm ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/postrm b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/postrm index 21a01fa..8de05c1 100644 --- a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/postrm +++ b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/postrm @@ -13,3 +13,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License + +LOGSEARCH_SCRIPT_LINK_NAME="/usr/bin/logsearch" +LOGSEARCH_CONF_DIR_LINK="/etc/ambari-logsearch-portal/conf" + +rm -f $LOGSEARCH_SCRIPT_LINK_NAME +rm -f $LOGSEARCH_CONF_DIR_LINK \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/preinst ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/preinst b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/preinst index 21a01fa..dfb8752 100644 --- a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/preinst +++ b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/portal/preinst @@ -13,3 +13,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License + +LOGSEARCH_CONF="/etc/ambari-logsearch-portal/conf" +LOGSEARCH_CONF_BACKUP="/usr/lib/ambari-logsearch-portal/conf-old" + +if [ -d $LOGSEARCH_CONF_BACKUP ]; then + rm -rf $LOGSEARCH_CONF_BACKUP +fi + +if [ -d $LOGSEARCH_CONF ]; then + mkdir -p $LOGSEARCH_CONF_BACKUP + cp -r $LOGSEARCH_CONF/* $LOGSEARCH_CONF_BACKUP + rm -rf $LOGSEARCH_CONF +fi \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/postinstall.sh ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/postinstall.sh b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/postinstall.sh new file mode 100755 index 0000000..a4eec77 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/postinstall.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +LOGFEEDER_SCRIPT_LINK_NAME="/usr/bin/logfeeder" +LOGFEEDER_SCRIPT_SOURCE="/usr/lib/ambari-logsearch-logfeeder/bin/logfeeder.sh" + +LOGFEEDER_CONF_LINK="/etc/ambari-logsearch-logfeeder/conf" +LOGFEEDER_CONF_SOURCE="/usr/lib/ambari-logsearch-logfeeder/conf" + +ln -s $LOGFEEDER_SCRIPT_SOURCE $LOGFEEDER_SCRIPT_LINK_NAME +ln -s $LOGFEEDER_CONF_SOURCE $LOGFEEDER_CONF_LINK + +# handle old checkpoint & keys folder + +LOGFEEDER_CONF_BACKUP="/usr/lib/ambari-logsearch-logfeeder/conf-old" + +if [ -d "$LOGFEEDER_CONF_BACKUP" ]; then + if [ -d "$LOGFEEDER_CONF_BACKUP/keys" ]; then + cp -r $LOGFEEDER_CONF_BACKUP/keys $LOGFEEDER_CONF_SOURCE + fi + if [ -d "$LOGFEEDER_CONF_BACKUP/checkpoints" ]; then + cp -r $LOGFEEDER_CONF_BACKUP/checkpoints $LOGFEEDER_CONF_SOURCE + fi +fi \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/postremove.sh ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/postremove.sh b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/postremove.sh new file mode 100755 index 0000000..5f1c623 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/postremove.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +LOGFEEDER_SCRIPT_LINK_NAME="/usr/bin/logfeeder" +LOGFEEDER_CONF_DIR_LINK="/etc/ambari-logsearch-logfeeder/conf" + +rm -f $LOGFEEDER_SCRIPT_LINK_NAME +rm -f $LOGFEEDER_CONF_DIR_LINK \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/preinstall.sh ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/preinstall.sh b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/preinstall.sh new file mode 100644 index 0000000..01d70e0 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/preinstall.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +LOGFEEDER_CONF="/etc/ambari-logsearch-logfeeder/conf" +LOGFEEDER_CONF_BACKUP="/usr/lib/ambari-logsearch-logfeeder/conf-old" + +if [ -d $LOGFEEDER_CONF_BACKUP ]; then + rm -rf $LOGFEEDER_CONF_BACKUP +fi + +if [ -d $LOGFEEDER_CONF ]; then + mkdir -p $LOGFEEDER_CONF_BACKUP + cp -r $LOGFEEDER_CONF/* $LOGFEEDER_CONF_BACKUP + rm -rf $LOGFEEDER_CONF +fi \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/portal/postinstall.sh ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/portal/postinstall.sh b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/portal/postinstall.sh new file mode 100644 index 0000000..9107482 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/portal/postinstall.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +LOGSEARCH_SCRIPT_LINK_NAME="/usr/bin/logsearch" +LOGSEARCH_SCRIPT_SOURCE="/usr/lib/ambari-logsearch-portal/bin/logsearch.sh" + +LOGSEARCH_CONF_LINK="/etc/ambari-logsearch-portal/conf" +LOGSEARCH_CONF_SOURCE="/usr/lib/ambari-logsearch-portal/conf" + +ln -s $LOGSEARCH_SCRIPT_SOURCE $LOGSEARCH_SCRIPT_LINK_NAME +ln -s $LOGSEARCH_CONF_SOURCE $LOGSEARCH_CONF_LINK + +# handle old keys folder + +LOGSEARCH_CONF_BACKUP="/usr/lib/ambari-logsearch-portal/conf-old" + +if [ -d "$LOGSEARCH_CONF_BACKUP" ]; then + if [ -d "$LOGSEARCH_CONF_BACKUP/keys" ]; then + cp -r $LOGSEARCH_CONF_BACKUP/keys $LOGSEARCH_CONF_SOURCE + fi +fi \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/portal/postremove.sh ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/portal/postremove.sh b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/portal/postremove.sh new file mode 100644 index 0000000..8de05c1 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/portal/postremove.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +LOGSEARCH_SCRIPT_LINK_NAME="/usr/bin/logsearch" +LOGSEARCH_CONF_DIR_LINK="/etc/ambari-logsearch-portal/conf" + +rm -f $LOGSEARCH_SCRIPT_LINK_NAME +rm -f $LOGSEARCH_CONF_DIR_LINK \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/portal/preinstall.sh ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/portal/preinstall.sh b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/portal/preinstall.sh new file mode 100644 index 0000000..dfb8752 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/portal/preinstall.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +LOGSEARCH_CONF="/etc/ambari-logsearch-portal/conf" +LOGSEARCH_CONF_BACKUP="/usr/lib/ambari-logsearch-portal/conf-old" + +if [ -d $LOGSEARCH_CONF_BACKUP ]; then + rm -rf $LOGSEARCH_CONF_BACKUP +fi + +if [ -d $LOGSEARCH_CONF ]; then + mkdir -p $LOGSEARCH_CONF_BACKUP + cp -r $LOGSEARCH_CONF/* $LOGSEARCH_CONF_BACKUP + rm -rf $LOGSEARCH_CONF +fi \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/README.md ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/README.md b/ambari-logsearch/ambari-logsearch-logfeeder/README.md index d2e55f0..48cb81b 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/README.md +++ b/ambari-logsearch/ambari-logsearch-logfeeder/README.md @@ -29,11 +29,11 @@ First you need to start every required service (except logfeeder), go to `ambari docker-compose up -d zookeeper solr logsearch ``` -Secondly, if you are planning to run Log Feeder from an IDE, for running the LogFeeder main methoud, you will need to set the working directory to `ambari/ambari-logsearch/ambari-logsearch-logfeeder` and use `--monitor` as a command line argument. +Secondly, if you are planning to run Log Feeder from an IDE, for running the LogFeeder main methoud, you will need to set the working directory to `ambari/ambari-logsearch/ambari-logsearch-logfeeder` or set `LOGFEEDER_RELATIVE_LOCATION` env variable. With Maven, you won't need these steps, just run this command from the ambari-logsearch-logfeeder folder: ```bash -mvn clean package -DskipTests exec:java +mvn clean package -DskipTests spring-boot:run ``` # Input Configuration http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/build.properties ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/build.properties b/ambari-logsearch/ambari-logsearch-logfeeder/build.properties deleted file mode 100644 index 013ba2e..0000000 --- a/ambari-logsearch/ambari-logsearch-logfeeder/build.properties +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -app.pkg.dir=${app.work.dir}/pkg - -app.dev.war.dir=${app.work.dir}/webapps/logsearch -app.war.name=logsearch.war - -app.target.dir=${builddir}/target/classes/static \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/build.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/build.xml b/ambari-logsearch/ambari-logsearch-logfeeder/build.xml index 738b2ef..c150b55 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/build.xml +++ b/ambari-logsearch/ambari-logsearch-logfeeder/build.xml @@ -19,10 +19,8 @@ <property environment="env"/> <property name="debuglevel" value="source,lines,vars"/> <dirname property="builddir" file="build.xml"/> - <property name="target" value="1.7"/> - <property name="source" value="1.7"/> - <property file="local.properties"/> - <property file="build.properties"/> + <property name="target" value="1.8"/> + <property name="source" value="1.8"/> <target name="init"> </target> <target name="build"/> @@ -31,18 +29,20 @@ <delete dir="target/package"/> <copy todir="target/package/libs" includeEmptyDirs="no"> <fileset dir="target/libs"/> + <fileset file="target/ambari-logsearch-logfeeder*.jar"/> </copy> - <copy todir="target/package/classes" includeEmptyDirs="no"> - <fileset dir="target/classes"/> + <copy todir="target/package/bin" includeEmptyDirs="no"> + <fileset file="src/main/scripts/logfeeder.sh"/> </copy> - <copy todir="target/package" includeEmptyDirs="no"> - <fileset file="config.json"/> + <copy todir="target/package/conf" includeEmptyDirs="no"> + <fileset file="target/classes/grok-patterns"/> + <fileset file="target/classes/log4j.xml"/> + <fileset file="target/classes/logfeeder.properties"/> + <fileset file="target/classes/alias_config.json"/> + <fileset file="src/main/scripts/logfeeder-env.sh"/> </copy> - <copy todir="target/package" includeEmptyDirs="no"> - <fileset file="grok-patterns"/> - <fileset file="src/main/scripts/run.sh"/> - </copy> - <chmod file="target/package/run.sh" perm="755"/> + <chmod file="target/package/bin/logfeeder.sh" perm="755"/> + <chmod file="target/package/conf/logfeeder-env.sh" perm="755"/> <tar compression="gzip" destfile="target/ambari-logsearch-logfeeder.tgz"> <tarfileset mode="755" dir="target/package"> <include name="*.sh"/> @@ -52,6 +52,4 @@ </tarfileset> </tar> </target> - <target description="Build all projects which reference this project. Useful to propagate changes." - name="build-refprojects"/> </project> http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml b/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml index 005af15..6a3524d 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml +++ b/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml @@ -198,7 +198,6 @@ </dependency> </dependencies> <build> - <finalName>LogFeeder</finalName> <pluginManagement> <plugins> <plugin> @@ -222,40 +221,10 @@ <target>${jdk.version}</target> </configuration> </plugin> - - <!-- Exec main class plugin --> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>exec-maven-plugin</artifactId> - <version>1.2.1</version> - <executions> - <execution> - <goals> - <goal>java</goal> - </goals> - </execution> - </executions> - <configuration> - <mainClass>org.apache.ambari.logfeeder.LogFeeder</mainClass> - <arguments> - <argument>--monitor</argument> - </arguments> - </configuration> - </plugin> <plugin> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-maven-plugin</artifactId> <version>${spring-boot.version}</version> - <configuration> - <classifier>exec</classifier> - </configuration> - <executions> - <execution> - <goals> - <goal>repackage</goal> - </goals> - </execution> - </executions> </plugin> <!-- copy-dependencies --> <plugin> http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/run.sh ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/run.sh b/ambari-logsearch/ambari-logsearch-logfeeder/run.sh index 70947ec..47a9e6c 100755 --- a/ambari-logsearch/ambari-logsearch-logfeeder/run.sh +++ b/ambari-logsearch/ambari-logsearch-logfeeder/run.sh @@ -14,6 +14,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -echo "LOG FEEDER" -mvn clean compile -mvn exec:java +mvn clean spring-boot:run http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeeder.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeeder.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeeder.java index 2d31e5a..235109c 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeeder.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeeder.java @@ -18,54 +18,21 @@ */ package org.apache.ambari.logfeeder; -import com.google.gson.GsonBuilder; -import org.apache.ambari.logfeeder.common.LogEntryParseTester; -import org.apache.commons.io.FileUtils; -import org.apache.log4j.LogManager; import org.springframework.boot.Banner; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.boot.system.ApplicationPidFileWriter; -import java.io.File; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - @SpringBootApplication( scanBasePackages = {"org.apache.ambari.logfeeder"} ) public class LogFeeder { public static void main(String[] args) { - LogFeederCommandLine cli = new LogFeederCommandLine(args); - if (cli.isTest()) { - test(cli); - } - String pidFile = System.getenv("PID_FILE") == null ? "logfeeder.pid" : System.getenv("PID_FILE"); + String pidFile = System.getenv("LOGFEEDER_PID_FILE") == null ? "logfeeder.pid" : System.getenv("LOGFEEDER_PID_FILE"); new SpringApplicationBuilder(LogFeeder.class) .bannerMode(Banner.Mode.OFF) .listeners(new ApplicationPidFileWriter(pidFile)) .run(args); } - - private static void test(LogFeederCommandLine cli) { - try { - LogManager.shutdown(); - String testLogEntry = cli.getTestLogEntry(); - String testShipperConfig = FileUtils.readFileToString(new File(cli.getTestShipperConfig()), Charset.defaultCharset()); - List<String> testGlobalConfigs = new ArrayList<>(); - for (String testGlobalConfigFile : cli.getTestGlobalConfigs().split(",")) { - testGlobalConfigs.add(FileUtils.readFileToString(new File(testGlobalConfigFile), Charset.defaultCharset())); - } - String testLogId = cli.getTestLogId(); - Map<String, Object> result = new LogEntryParseTester(testLogEntry, testShipperConfig, testGlobalConfigs, testLogId).parse(); - String parsedLogEntry = new GsonBuilder().setPrettyPrinting().create().toJson(result); - System.out.println("The result of the parsing is:\n" + parsedLogEntry); - } catch (Exception e) { - System.out.println("Exception occurred, could not test if log entry is parseable"); - e.printStackTrace(System.out); - } - } } http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederCommandLine.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederCommandLine.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederCommandLine.java index 61e7a1e..a6fcb2a 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederCommandLine.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederCommandLine.java @@ -19,22 +19,25 @@ package org.apache.ambari.logfeeder; +import com.google.gson.GsonBuilder; +import org.apache.ambari.logfeeder.common.LogEntryParseTester; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.log4j.LogManager; +import java.io.File; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; +import java.util.Map; public class LogFeederCommandLine { - - private static final Logger LOG = LoggerFactory.getLogger(LogFeederCommandLine.class); private static final String TEST_COMMAND = "test"; private static final String TEST_LOG_ENTRY_OPTION = "test-log-entry"; @@ -42,11 +45,9 @@ public class LogFeederCommandLine { private static final String TEST_GLOBAL_CONFIG_OPTION = "test-global-config"; private static final String TEST_LOG_ID_OPTION = "test-log-id"; - private static final String COMMAND_LINE_SYNTAX = "java org.apache.ambari.logfeeder.LogFeeder -(monitor|test) [args]"; - - private CommandLine cli; + private static final String COMMAND_LINE_SYNTAX = "java org.apache.ambari.logfeeder.LogFeederCommandLine --test [args]"; - public LogFeederCommandLine(String[] args) { + public static void main(String[] args) { Options options = new Options(); HelpFormatter helpFormatter = new HelpFormatter(); helpFormatter.setDescPadding(10); @@ -95,7 +96,7 @@ public class LogFeederCommandLine { try { CommandLineParser cmdLineParser = new DefaultParser(); - cli = cmdLineParser.parse(options, args); + CommandLine cli = cmdLineParser.parse(options, args); if (cli.hasOption('h')) { helpFormatter.printHelp(COMMAND_LINE_SYNTAX, options); @@ -105,11 +106,12 @@ public class LogFeederCommandLine { if (cli.hasOption("t")) { command = TEST_COMMAND; validateRequiredOptions(cli, command, testLogEntryOption, testShipperConfOption); - } else { - LOG.info("Start application in monitor mode "); } + test(cli); } catch (Exception e) { - LOG.info("Error parsing command line parameters: {}. LogFeeder will be started in monitoring mode.", e.getMessage()); + e.printStackTrace(); + helpFormatter.printHelp(COMMAND_LINE_SYNTAX, options); + System.exit(1); } } @@ -125,24 +127,23 @@ public class LogFeederCommandLine { String.format("The following options required for '%s' : %s", command, StringUtils.join(requiredOptions, ","))); } } - - public boolean isTest() { - return cli != null && cli.hasOption('t'); - } - - public String getTestLogEntry() { - return cli.getOptionValue("tle"); - } - - public String getTestShipperConfig() { - return cli.getOptionValue("tsc"); - } - - public String getTestGlobalConfigs() { - return cli.getOptionValue("tgc"); - } - - public String getTestLogId() { - return cli.getOptionValue("tli"); + + private static void test(CommandLine cli) { + try { + LogManager.shutdown(); + String testLogEntry = cli.getOptionValue("tle"); + String testShipperConfig = FileUtils.readFileToString(new File(cli.getOptionValue("tsc")), Charset.defaultCharset()); + List<String> testGlobalConfigs = new ArrayList<>(); + for (String testGlobalConfigFile : cli.getOptionValue("tgc").split(",")) { + testGlobalConfigs.add(FileUtils.readFileToString(new File(testGlobalConfigFile), Charset.defaultCharset())); + } + String testLogId = cli.getOptionValue("tli"); + Map<String, Object> result = new LogEntryParseTester(testLogEntry, testShipperConfig, testGlobalConfigs, testLogId).parse(); + String parsedLogEntry = new GsonBuilder().setPrettyPrinting().create().toJson(result); + System.out.println("The result of the parsing is:\n" + parsedLogEntry); + } catch (Exception e) { + System.out.println("Exception occurred, could not test if log entry is parseable"); + e.printStackTrace(System.out); + } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java index 35c0e6a..037f4a1 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java @@ -21,12 +21,12 @@ package org.apache.ambari.logfeeder.common; import java.io.BufferedInputStream; import java.io.File; +import java.io.FileNotFoundException; import java.lang.reflect.Type; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -62,6 +62,7 @@ import org.apache.log4j.Level; import org.apache.log4j.Logger; import com.google.gson.reflect.TypeToken; +import org.springframework.core.io.ClassPathResource; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -146,6 +147,10 @@ public class ConfigHandler implements InputConfigMonitor { private void loadConfigsUsingClassLoader(String configFileName) throws Exception { try (BufferedInputStream fis = (BufferedInputStream) this.getClass().getClassLoader().getResourceAsStream(configFileName)) { + ClassPathResource configFile = new ClassPathResource(configFileName); + if (!configFile.exists()) { + throw new FileNotFoundException(configFileName); + } String configData = IOUtils.toString(fis, Charset.defaultCharset()); loadConfigs(configData); } http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/control ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/control b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/control deleted file mode 100644 index 40cd855..0000000 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/control +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License -Package: [[artifactId]] -Version: [[package-version]]-[[package-release]] -Section: [[deb.section]] -Priority: [[deb.priority]] -Depends: [[deb.dependency.list]] -Architecture: [[deb.architecture]] -Description: [[description]] -Maintainer: [[deb.publisher]] http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postinst ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postinst b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postinst deleted file mode 100644 index 21a01fa..0000000 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postinst +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postrm ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postrm b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postrm deleted file mode 100644 index 21a01fa..0000000 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/postrm +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/preinst ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/preinst b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/preinst deleted file mode 100644 index 21a01fa..0000000 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/preinst +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/prerm ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/prerm b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/prerm deleted file mode 100644 index 21a01fa..0000000 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/package/deb/control/prerm +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/logfeeder-env.sh ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/logfeeder-env.sh b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/logfeeder-env.sh new file mode 100755 index 0000000..37e9930 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/logfeeder-env.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Log Feeder extra options +export LOGFEEDER_JAVA_OPTS=${LOGFEEDER_JAVA_OPTS:-""} + +# Log Feeder debug options +# export LOGFEEDER_DEBUG=true +# exoprt LOGFEEDER_DEBUG_SUSPEND=n +export LOGFEEDER_DEBUG_PORT=5006 + +# Log Feeder memory +# export LOGFEEDER_JAVA_MEM="-Xmx512m" + +# export LOG_PATH=/var/log/ambari-logsearch-logfeeder/ +# export LOG_FILE=logfeeder.log + +# Pid file of the application +# export LOGFEEDER_PID_DIR=/var/run/ambari-logsearch-logfeeder +# export LOGFEEDER_PID_FILE=logfeeder.pid + +# SSL settings" +# export LOGFEEDER_SSL="true" +# export LOGFEEDER_KEYSTORE_LOCATION="/my/path/keystore.jks" +# export LOGFEEDER_KEYSTORE_TYPE="jks" +# export LOGFEEDER_TRUSTSTORE_LOCATION="/my/path/trutstore.jks" +# export LOGFEEDER_TRUSTSTORE_TYPE="jks" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh new file mode 100755 index 0000000..cfacea5 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh @@ -0,0 +1,292 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +JVM="java" + +if [ -x $JAVA_HOME/bin/java ]; then + JVM=$JAVA_HOME/bin/java +fi + +if [ "$LOGFEEDER_JAVA_MEM" = "" ]; then + LOGFEEDER_JAVA_MEM="-Xmx512m" +fi + +readlinkf(){ + # get real path on mac OSX + perl -MCwd -e 'print Cwd::abs_path shift' "$1"; +} + +if [ "$(uname -s)" = 'Linux' ]; then + SCRIPT_DIR="`dirname "$(readlink -f "$0")"`" +else + SCRIPT_DIR="`dirname "$(readlinkf "$0")"`" +fi + +LOGFEEDER_ROOT_DIR="`dirname \"$SCRIPT_DIR\"`" +LOGFEEDER_LIBS_DIR="$LOGFEEDER_ROOT_DIR/libs" + +if [ "$LOGFEEDER_CONF_DIR" = "" ]; then + LOGFEEDER_CONF_DIR="/etc/ambari-logsearch-logfeeder/conf" + if [ ! -d $LOGFEEDER_CONF_DIR ]; then + if [ -d "$LOGFEEDER_ROOT_DIR/conf" ]; then + LOGFEEDER_CONF_DIR="$LOGFEEDER_ROOT_DIR/conf" + fi + fi +fi + +if [ -f "$LOGFEEDER_CONF_DIR/logfeeder-env.sh" ]; then + source $LOGFEEDER_CONF_DIR/logfeeder-env.sh +fi + +if [ ! -z "$LOGSEARCH_SOLR_CLIENT_SSL_INCLUDE" ]; then + source $LOGSEARCH_SOLR_CLIENT_SSL_INCLUDE +fi + +if [ -z "$LOGFEEDER_PID_FILE" ]; then + LOGFEEDER_PID_DIR=$HOME + export LOGFEEDER_PID_FILE=$LOGFEEDER_PID_DIR/logfeeder.pid +fi + +if [ -z "$LOG_FILE" ]; then + export LOG_FILE="logfeeder.log" +fi + +LOGFEEDER_GC_LOGFILE="logfeeder_gc.log" + +if [ -z "$LOG_PATH" ]; then + LOG_FILE="$HOME/$LOG_FILE" + LOGFEEDER_GC_LOGFILE="$HOME/$LOGFEEDER_GC_LOGFILE" +else + LOG_PATH_WITHOUT_SLASH=${LOG_PATH%/} + LOG_FILE="$LOG_PATH_WITHOUT_SLASH/$LOG_FILE" + LOGFEEDER_GC_LOGFILE="$LOG_PATH_WITHOUT_SLASH/$LOGFEEDER_GC_LOGFILE" +fi + +LOGFEEDER_GC_OPTS="-XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$LOGFEEDER_GC_LOGFILE" + +function print_usage() { + cat << EOF + + Usage: [<command>] [<arguments with flags>] + + commands: + start Start Log Feeder + stop Stop Log Feeder + status Check Log Feeder status (pid file) + test Test Log Feeder shipper configs + help Print usage + + + start command arguments: + -d, --debug Start java process in debug mode + -f, --foreground Start java process in foreground + + test command arguments: + -h, --help Print usage + -tle, --test-log-entry Log entry to test if it's parseable (required) + -tsc, --test-shipper-config Shipper configuration file for testing if log entry is parseable (required) + -tgc, --test-global-config Global configuration files (comma separated list) for testing if log entry is parseable + -tli, --test-log-id The id of the log to test + +EOF +} + +function spinner() { + local pid=$1 + local delay=0.5 + local spinstr='|/-\' + while [ "$(ps aux | awk '{print $2}' | grep -w $pid)" ]; do + local temp=${spinstr#?} + printf " [%c] " "$spinstr" + local spinstr=$temp${spinstr%"$temp"} + sleep $delay + printf "\b\b\b\b\b\b" + done + printf " \b\b\b\b" +} + +function status() { + echo "Checking Log Feeder status ..." + if [ -f "$LOGFEEDER_PID_FILE" ]; then + LOGFEEDER_PID=`cat "$LOGFEEDER_PID_FILE"` + else + echo "Log Feeder pid not exists. (probably the process is not running)" + exit 1 + fi + + if ps -p $LOGFEEDER_PID > /dev/null + then + echo "Log Feeder process is running. (pid: $LOGFEEDER_PID)" + exit 0 + else + echo "Log Feeder process is not running." + exit 1 + fi +} + +function start() { + (status); + exit_status=$(echo $?) + if [ "$exit_status" = "0" ]; then + echo "Skipping start process." + exit 0 + fi + + FG="false" + LOGFEEDER_DEBUG_SUSPEND=${LOGFEEDER_DEBUG_SUSPEND:-n} + LOGFEEDER_DEBUG_PORT=${LOGFEEDER_DEBUG_PORT:-"5006"} + + if [ "$LOGFEEDER_DEBUG" = "true" ]; then + LOGFEEDER_JAVA_OPTS="$LOGFEEDER_JAVA_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,address=$LOGFEEDER_DEBUG_PORT,server=y,suspend=$LOGFEEDER_DEBUG_SUSPEND " + fi + + if [ "$LOGFEEDER_SSL" = "true" ]; then + LOGFEEDER_JAVA_OPTS="$LOGFEEDER_JAVA_OPTS -Djavax.net.ssl.keyStore=$LOGFEEDER_KEYSTORE_LOCATION -Djavax.net.ssl.keyStoreType=$LOGFEEDER_KEYSTORE_TYPE -Djavax.net.ssl.trustStore=$LOGFEEDER_TRUSTSTORE_LOCATION -Djavax.net.ssl.trustStoreType=$LOGFEEDER_TRUSTSTORE_TYPE" + fi + + if [ "$LOGFEEDER_JMX" = "true" ]; then + LOGFEEDER_JAVA_OPTS="$LOGFEEDER_JAVA_OPTS -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=2098" + fi + + if [ $# -gt 0 ]; then + while true; do + case "$1" in + -f|--foreground) + FG="true" + shift + ;; + -d|--debug) + if [ "$LOGFEEDER_DEBUG" != "true" ]; then + LOGFEEDER_JAVA_OPTS="$LOGFEEDER_JAVA_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,address=$LOGFEEDER_DEBUG_PORT,server=y,suspend=$LOGFEEDER_DEBUG_SUSPEND " + fi + shift + ;; + *) + if [ "${1:0:2}" == "-D" ]; then + # pass thru any opts that begin with -D (java system props) + LOGFEEDER_JAVA_OPTS+=("$1") + echo "$LOGFEEDER_JAVA_OPTS" + shift + else + if [ "$1" != "" ]; then + print_usage + exit 1 + else + break + fi + fi + ;; + esac + done + fi + + if [ $FG == "true" ]; then + echo "Starting logfeeder... (foreground) pid_file=$LOGFEEDER_PID_FILE" + echo "Run command $JVM -cp '$LOGFEEDER_CONF_DIR:$LOGFEEDER_LIBS_DIR/*' $LOGFEEDER_GC_OPTS $LOGFEEDER_JAVA_OPTS $LOGFEEDER_JAVA_MEM org.apache.ambari.logfeeder.LogFeeder" + $JVM -cp "$LOGFEEDER_CONF_DIR:$LOGFEEDER_LIBS_DIR/*" $LOGFEEDER_GC_OPTS $LOGFEEDER_JAVA_OPTS $LOGFEEDER_JAVA_MEM org.apache.ambari.logfeeder.LogFeeder + else + echo "Starting logfeeder... Output file=$LOG_FILE pid_file=$LOGFEEDER_PID_FILE" + echo "Run command nohup $JVM -cp '$LOGFEEDER_CONF_DIR:$LOGFEEDER_LIBS_DIR/*' $LOGFEEDER_GC_OPTS $LOGFEEDER_JAVA_OPTS $LOGFEEDER_JAVA_MEM org.apache.ambari.logfeeder.LogFeeder" + nohup $JVM -cp "$LOGFEEDER_CONF_DIR:$LOGFEEDER_LIBS_DIR/*" $LOGFEEDER_GC_OPTS $LOGFEEDER_JAVA_OPTS $LOGFEEDER_JAVA_MEM org.apache.ambari.logfeeder.LogFeeder > $LOG_FILE 2>&1 & + fi +} + +function stop() { + LOGFEEDER_STOP_WAIT=3 + if [ -f "$LOGFEEDER_PID_FILE" ]; then + LOGFEEDER_PID=`cat "$LOGFEEDER_PID_FILE"` + fi + + if [ "$LOGFEEDER_PID" != "" ]; then + echo -e "Sending stop command to Log Feeder... Checking PID: $LOGFEEDER_PID." + kill $LOGFEEDER_PID + (loops=0 + while true + do + CHECK_PID=`ps auxww | awk '{print $2}' | grep -w $LOGFEEDER_PID | sort -r | tr -d ' '` + if [ "$CHECK_PID" != "" ]; then + slept=$((loops * 2)) + if [ $slept -lt $LOGFEEDER_STOP_WAIT ]; then + sleep 2 + loops=$[$loops+1] + else + exit # subshell! + fi + else + exit # subshell! + fi + done) & + spinner $! + rm -f "$LOGFEEDER_PID_FILE" + else + echo -e "No LogFeeder process found to stop." + exit 0 + fi + + CHECK_PID=`ps auxww | awk '{print $2}' | grep -w $LOGFEEDER_PID | sort -r | tr -d ' '` + if [ "$CHECK_PID" != "" ]; then + echo -e "Log Feeder process $LOGFEEDER_PID is still running; forcefully killing it now." + kill -9 $LOGFEEDER_PID + echo "Killed process $LOGFEEDER_PID" + rm -f "$LOGFEEDER_PID_FILE" + sleep 1 + else + echo "Log Feeder is stopped." + fi + + CHECK_PID=`ps auxww | awk '{print $2}' | grep -w $LOGFEEDER_PID | sort -r | tr -d ' '` + if [ "$CHECK_PID" != "" ]; then + echo "ERROR: Failed to kill Log Feeder Java process $LOGFEEDER_PID ... script fails." + exit 1 + fi +} + +function test() { + echo "Running command: $JVM -cp "$LOGFEEDER_CONF_DIR:$LOGFEEDER_LIBS_DIR/*" org.apache.ambari.logfeeder.LogFeederCommandLine --test ${@}" + $JVM -cp "$LOGFEEDER_CONF_DIR:$LOGFEEDER_LIBS_DIR/*" $LOGFEEDER_JAVA_OPTS org.apache.ambari.logfeeder.LogFeederCommandLine --test ${@} +} + +if [ $# -gt 0 ]; then + SCRIPT_CMD="$1" + shift +else + print_usage + exit 1 +fi + +case $SCRIPT_CMD in + start) + start ${1+"$@"} + ;; + stop) + stop + ;; + status) + status + ;; + test) + test ${1+"$@"} + ;; + help) + print_usage + exit 0 + ;; + *) + print_usage + exit 1 + ;; + +esac \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh deleted file mode 100644 index 22e1cb2..0000000 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -curr_dir=`pwd` -cd `dirname $0`; script_dir=`pwd`; cd $curr_dir - -foreground=0 -if [ "$1" = "-foreground" ]; then - foreground=1 - shift -fi - -if [ ! -z "$LOGFEEDER_INCLUDE" ]; then - source $LOGFEEDER_INCLUDE -fi - -if [ ! -z "$LOGSEARCH_SOLR_CLIENT_SSL_INCLUDE" ]; then - source $LOGSEARCH_SOLR_CLIENT_SSL_INCLUDE -fi - -JAVA=java -if [ -x $JAVA_HOME/bin/java ]; then - JAVA=$JAVA_HOME/bin/java -fi - -if [ "$LOGFEEDER_JAVA_MEM" = "" ]; then - LOGFEEDER_JAVA_MEM="-Xmx512m" -fi - -if [ "$LOGFILE" = "" ]; then - LOGFILE="/var/log/logfeeder/logfeeder.out" -fi - -if [ "$PID_FILE" = "" ]; then - LOGFEEDER_PID_DIR=$HOME - PID_FILE=$LOGFEEDER_PID_DIR/logsearch-logfeeder-$USER.pid -fi - -if [ "$LOGFEEDER_CONF_DIR" = "" ]; then - LOGFEEDER_CONF_DIR="/etc/logfeeder/conf" - if [ ! -d $LOGFEEDER_CONF_DIR ]; then - if [ -d $script_dir/classes ]; then - LOGFEEDER_CONF_DIR=$script_dir/classes - fi - fi -fi - -LOGFEEDER_DEBUG_SUSPEND=${LOGFEEDER_DEBUG_SUSPEND:-n} -if [ "$LOGFEEDER_DEBUG" = "true" ] && [ ! -z "$LOGFEEDER_DEBUG_PORT" ]; then - LOGFEEDER_JAVA_OPTS="$LOGFEEDER_JAVA_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,address=$LOGFEEDER_DEBUG_PORT,server=y,suspend=$LOGFEEDER_DEBUG_SUSPEND " -fi - -LOGFEEDER_GC_LOGFILE=`dirname $LOGFILE`/logfeeder_gc.log -LOGFEEDER_GC_OPTS="-XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$LOGFEEDER_GC_LOGFILE" - -#JMX="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=2098" - -if [ "$LOGFEEDER_SSL" = "true" ]; then - LOGFEEDER_JAVA_OPTS="$LOGFEEDER_JAVA_OPTS -Djavax.net.ssl.keyStore=$LOGFEEDER_KEYSTORE_LOCATION -Djavax.net.ssl.keyStoreType=$LOGFEEDER_KEYSTORE_TYPE -Djavax.net.ssl.trustStore=$LOGFEEDER_TRUSTSTORE_LOCATION -Djavax.net.ssl.trustStoreType=$LOGFEEDER_TRUSTSTORE_TYPE" -fi - -if [ $foreground -eq 0 ]; then - if [ -f ${PID_FILE} ]; then - PID=`cat ${PID_FILE}` - if kill -0 $PID 2>/dev/null; then - echo "logfeeder already running (${PID}) killing..." - kill $PID 2>/dev/null - sleep 5 - if kill -0 $PID 2>/dev/null; then - echo "logfeeder still running. Will kill process forcefully in another 10 seconds..." - sleep 10 - kill -9 $PID 2>/dev/null - sleep 2 - fi - fi - - if kill -0 $PID 2>/dev/null; then - echo "ERROR: Even after all efforts to stop logfeeder, it is still running. pid=$PID. Please manually kill the service and try again." - exit 1 - fi - fi - - echo "Starting logfeeder. Output file=$LOGFILE pid_file=$PID_FILE" - #LOGFEEDER_CLI_CLASSPATH=set -x -nohup $JAVA -cp "$LOGFEEDER_CLI_CLASSPATH:$LOGFEEDER_CONF_DIR:$script_dir/libs/*:$script_dir/classes" $LOGFEEDER_GC_OPTS $LOGFEEDER_JAVA_MEM $LOGFEEDER_JAVA_OPTS $JMX org.apache.ambari.logfeeder.LogFeeder --monitor $* > $LOGFILE 2>&1 & - echo $! > $PID_FILE -else -$JAVA -cp "$LOGFEEDER_CLI_CLASSPATH:$LOGFEEDER_CONF_DIR:$script_dir/libs/*:$script_dir/classes" $LOGFEEDER_JAVA_MEM $LOGFEEDER_JAVA_OPTS $JMX org.apache.ambari.logfeeder.LogFeeder --monitor $* -fi - http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-server/README.md ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/README.md b/ambari-logsearch/ambari-logsearch-server/README.md index 26b1f73..7432344 100644 --- a/ambari-logsearch/ambari-logsearch-server/README.md +++ b/ambari-logsearch/ambari-logsearch-server/README.md @@ -33,7 +33,7 @@ Then you can start Log Search server from maven cd ambari/ambari-logsearch/ambari-logsearch-server ./run.sh # or -mvn clean spring-boot:run +mvn clean package -DskipTests spring-boot:run ``` You can also start Log Search server from an IDE as well. One thing is important: the config set location that the server tries to upload to ZooKeeper. By default config sets are located at `${LOGSEARCH_SERVER_RELATIVE_LOCATION:}src/main/configsets` in `logsearch.properties`. Based or from where you run `LogSearch.java`, you need to set `LOGSEARCH_SERVER_RELATIVE_LOCATION` env variable properly. http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-server/build.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/build.xml b/ambari-logsearch/ambari-logsearch-server/build.xml index 5c50485..c711fe1 100644 --- a/ambari-logsearch/ambari-logsearch-server/build.xml +++ b/ambari-logsearch/ambari-logsearch-server/build.xml @@ -19,9 +19,8 @@ <property environment="env"/> <property name="debuglevel" value="source,lines,vars"/> <dirname property="builddir" file="build.xml"/> - <property name="target" value="1.7"/> - <property name="source" value="1.7"/> - <property file="build.properties"/> + <property name="target" value="1.8"/> + <property name="source" value="1.8"/> <target name="init"> </target> @@ -31,20 +30,34 @@ <delete dir="target/package"/> <copy todir="target/package/libs" includeEmptyDirs="no"> <fileset dir="target/libs"/> + <fileset file="target/ambari-logsearch-server*.jar"/> </copy> <copy todir="target/classes/static" includeEmptyDirs="no"> <fileset dir="target/dist"/> </copy> - <copy todir="target/package/classes" includeEmptyDirs="no"> - <fileset dir="target/classes"/> + <copy todir="target/package/webapp/static" includeEmptyDirs="no"> + <fileset dir="target/classes/static"/> </copy> - <copy todir="target/package/solr_configsets" includeEmptyDirs="yes"> + <copy todir="target/package/webapp/swagger" includeEmptyDirs="no"> + <fileset dir="target/classes/swagger"/> + </copy> + <copy todir="target/package/conf" includeEmptyDirs="no"> + <fileset file="src/main/scripts/logsearch-env.sh"/> + <fileset file="target/classes/log4j.xml"/> + <fileset file="target/classes/default.properties"/> + <fileset file="target/classes/logsearch.properties"/> + <fileset file="target/classes/info.properties"/> + <fileset file="target/classes/user_pass.json"/> + <fileset file="target/classes/HadoopServiceConfig.json"/> + </copy> + <copy todir="target/package/conf/solr_configsets" includeEmptyDirs="yes"> <fileset dir="src/main/configsets"/> </copy> - <copy todir="target/package" includeEmptyDirs="no"> - <fileset file="src/main/scripts/*"/> + <copy todir="target/package/bin" includeEmptyDirs="no"> + <fileset file="src/main/scripts/logsearch.sh"/> </copy> - <chmod file="target/package/*.sh" perm="755"/> + <chmod file="target/package/bin/logsearch.sh" perm="755"/> + <chmod file="target/package/conf/logsearch-env.sh" perm="755"/> <tar compression="gzip" destfile="target/ambari-logsearch-portal.tar.gz"> <tarfileset mode="755" dir="target/package"> <include name="*.sh"/> @@ -53,10 +66,5 @@ <exclude name="*.sh"/> </tarfileset> </tar> - </target> - - - <target description="Build all projects which reference this project. Useful to propagate changes." - name="build-refprojects"/> </project> http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-server/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/pom.xml b/ambari-logsearch/ambari-logsearch-server/pom.xml index 5444b00..6a7debf 100755 --- a/ambari-logsearch/ambari-logsearch-server/pom.xml +++ b/ambari-logsearch/ambari-logsearch-server/pom.xml @@ -44,7 +44,6 @@ <activeByDefault>true</activeByDefault> </activation> <build> - <finalName>LogSearch</finalName> <resources> <resource> <directory>src/main/resources</directory> @@ -77,16 +76,6 @@ <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-maven-plugin</artifactId> <version>${spring-boot.version}</version> - <configuration> - <classifier>exec</classifier> - </configuration> - <executions> - <execution> - <goals> - <goal>repackage</goal> - </goals> - </execution> - </executions> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-server/run.sh ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/run.sh b/ambari-logsearch/ambari-logsearch-server/run.sh index f245930..2de0536 100755 --- a/ambari-logsearch/ambari-logsearch-server/run.sh +++ b/ambari-logsearch/ambari-logsearch-server/run.sh @@ -17,4 +17,4 @@ echo " ââââââââââââââââââââââââââ âââââââââââââââââââ ââââââ ââââââââââââââ âââ ââââââââ âââââââ âââââââ âââââââââââââââââââ ââââââ âââ ââââââââââ âââ " -mvn clean spring-boot:run +mvn clean package -DskipTests spring-boot:run http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/LogSearch.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/LogSearch.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/LogSearch.java index 39343c7..45cda0a 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/LogSearch.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/LogSearch.java @@ -36,7 +36,7 @@ public class LogSearch { public static void main(String[] args) { - String pidFile = System.getenv("PID_FILE") == null ? "logsearch.pid" : System.getenv("PID_FILE"); + String pidFile = System.getenv("LOGSEARCH_PID_FILE") == null ? "logsearch.pid" : System.getenv("LOGSEARCH_PID_FILE"); new SpringApplicationBuilder(LogSearch.class) .bannerMode(Banner.Mode.OFF) .listeners(new ApplicationPidFileWriter(pidFile)) http://git-wip-us.apache.org/repos/asf/ambari/blob/562795c4/ambari-logsearch/ambari-logsearch-server/src/main/scripts/logsearch-env.sh ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/scripts/logsearch-env.sh b/ambari-logsearch/ambari-logsearch-server/src/main/scripts/logsearch-env.sh new file mode 100755 index 0000000..d8a837f --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/scripts/logsearch-env.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Log Search extra options +export LOGSEARCH_JAVA_OPTS=${LOGSEARCH_JAVA_OPTS:-""} + +# Log Search debug options +# export LOGSEARCH_DEBUG=true +# exoprt LOGSEARCH_DEBUG_SUSPEND=n +export LOGSEARCH_DEBUG_PORT=5005 + +# Log Search memory +# export LOGSEARCH_JAVA_MEM="--Xmx1024m" + +# export LOG_PATH=/var/log/ambari-logsearch-logfeeder/ +# export LOG_FILE=logsearch.log + +# Pid file of the application +# export LOGSEARCH_PID_DIR=/var/run/ambari-logsearch-logfeeder +# export LOGSEARCH_PID_FILE=logfeeder.pid + +# SSL settings" +# export LOGSEARCH_SSL="true" +# export LOGSEARCH_KEYSTORE_LOCATION="/my/path/keystore.jks" +# export LOGSEARCH_KEYSTORE_TYPE="jks" +# export LOGSEARCH_TRUSTSTORE_LOCATION="/my/path/trutstore.jks" +# export LOGSEARCH_TRUSTSTORE_TYPE="jks" \ No newline at end of file
