DRILL-6068: Support user/distrib-specific config checks during startup 1. Allows for distrib/user specific checks to be done 2. Place-holder files for distribution and user specific checks 3. Moved JVM Version Check to head of script Separation of setups into 3 levels
1. Apache Drill (Default) 2. Distribition Specific 3. User-Defined 1 & 2 are mutually exclusive. 3 is additional checks that the user can specify. Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/266250cf Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/266250cf Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/266250cf Branch: refs/heads/master Commit: 266250cf6ca2269f8772796d6c4eb903aa795111 Parents: 27aa236 Author: Kunal Khatua <kkha...@maprtech.com> Authored: Thu Jan 11 15:45:04 2018 -0800 Committer: Aman Sinha <asi...@maprtech.com> Committed: Fri Feb 23 17:55:06 2018 -0800 ---------------------------------------------------------------------- distribution/src/assemble/bin.xml | 15 ++ distribution/src/resources/auto-setup.sh | 29 ++++ distribution/src/resources/distrib-setup.sh | 29 ++++ distribution/src/resources/drill-config.sh | 178 +++++++++++++++-------- distribution/src/resources/drill-setup.sh | 26 ++++ 5 files changed, 217 insertions(+), 60 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/266250cf/distribution/src/assemble/bin.xml ---------------------------------------------------------------------- diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/bin.xml index b4290bb..bed34ed 100644 --- a/distribution/src/assemble/bin.xml +++ b/distribution/src/assemble/bin.xml @@ -347,6 +347,21 @@ <outputDirectory>conf</outputDirectory> </file> <file> + <source>src/resources/auto-setup.sh</source> + <fileMode>0755</fileMode> + <outputDirectory>bin</outputDirectory> + </file> + <file> + <source>src/resources/drill-setup.sh</source> + <fileMode>0755</fileMode> + <outputDirectory>conf</outputDirectory> + </file> + <file> + <source>src/resources/distrib-setup.sh</source> + <fileMode>0755</fileMode> + <outputDirectory>conf</outputDirectory> + </file> + <file> <source>src/resources/drill-override-example.conf</source> <outputDirectory>conf</outputDirectory> </file> http://git-wip-us.apache.org/repos/asf/drill/blob/266250cf/distribution/src/resources/auto-setup.sh ---------------------------------------------------------------------- diff --git a/distribution/src/resources/auto-setup.sh b/distribution/src/resources/auto-setup.sh new file mode 100644 index 0000000..75bdda0 --- /dev/null +++ b/distribution/src/resources/auto-setup.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is invoked by drill-config.sh during a Drillbit startup and provides +# default checks and autoconfiguration. +# Distributions should not put anything in this file. Checks can be +# specified in ${DRILL_HOME}/conf/distrib-setup.sh +# Users should not put anything in this file. Additional checks can be defined +# and put in ${DRILL_CONF_DIR}/drill-setup.sh instead. +# To FAIL any check, return with a non-zero return code +# e.g. +# if [ $status == "FAILED" ]; return 1; fi + +###========================================================================== +# FEATURES (Added here to describe supported additions) +###========================================================================== http://git-wip-us.apache.org/repos/asf/drill/blob/266250cf/distribution/src/resources/distrib-setup.sh ---------------------------------------------------------------------- diff --git a/distribution/src/resources/distrib-setup.sh b/distribution/src/resources/distrib-setup.sh new file mode 100644 index 0000000..dbd2b90 --- /dev/null +++ b/distribution/src/resources/distrib-setup.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is empty by default. Drill startup checks will be +# performed in drill-config.sh. Distributions can replace this file with a +# distribution-specific version that checks (and sets) environment variables +# and options specific to that distribution. +# Users should not put anything in this file. Additional checks can be defined +# and put in drill-setup.sh instead. +# To FAIL any check, return with a non-zero return code +# e.g. +# if [ $status == "FAILED" ]; return 1; fi + +###========================================================================== +# FEATURES (Added here to describe supported additions) +###========================================================================== http://git-wip-us.apache.org/repos/asf/drill/blob/266250cf/distribution/src/resources/drill-config.sh ---------------------------------------------------------------------- diff --git a/distribution/src/resources/drill-config.sh b/distribution/src/resources/drill-config.sh index 2341540..d92031f 100644 --- a/distribution/src/resources/drill-config.sh +++ b/distribution/src/resources/drill-config.sh @@ -64,6 +64,77 @@ while [ -h "$this" ]; do fi done +# JVM and Operating System Platform information +# Test for cygwin +is_cygwin=false +case "`uname`" in +CYGWIN*) is_cygwin=true;; +esac + +if $is_cygwin; then + JAVA_BIN="java.exe" +else + JAVA_BIN="java" +fi + +# Test for or find JAVA_HOME + +if [ -z "$JAVA_HOME" ]; then + SOURCE=`which java` + if [ -e $SOURCE ]; then + while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + # if $SOURCE was a relative symlink, we need to resolve it relative + # to the path where the symlink file was located + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + done + JAVA_HOME="$( cd -P "$( dirname "$SOURCE" )" && cd .. && pwd )" + JAVA=$SOURCE + fi + # if we didn't set it + if [ -z "$JAVA_HOME" ]; then + fatal_error "JAVA_HOME is not set and Java could not be found" + fi +fi + +# Now, verify that 'java' binary exists and is suitable for Drill. +# If we started with `which java` above, use that path (after replacing +# symlinks.) If we started with JAVA_HOME, try in bin. Doing so handles +# the case in which JAVA_HOME is a JDK that has a nested JRE; we prefer +# the JDK bin. Finally, if nothing else works, just search for the +# executable. + +if [ -z "$JAVA" ]; then + JAVA="$JAVA_HOME/bin/$JAVA_BIN" + if [[ ! -e $JAVA ]]; then + JAVA=`find -L "$JAVA_HOME" -name $JAVA_BIN -type f | head -n 1` + fi +fi +if [ ! -e "$JAVA" ]; then + fatal_error "Java not found at JAVA_HOME=$JAVA_HOME." +fi + +# Ensure that Java version is at least 1.7 +"$JAVA" -version 2>&1 | grep "version" | egrep -e "1\.4|1\.5|1\.6" > /dev/null +if [ $? -eq 0 ]; then + fatal_error "Java 1.7 or later is required to run Apache Drill." +fi + +# Check if a file exists and has relevant lines for execution +# Help in deciding which of the auto scripts to execute: distrib/drill +function checkExecutableLineCount() { + local lineCount=0 + #Empty input + if [ -z "$1" ]; then echo $lineCount; return; fi + #Not valid file + if [ ! -f "$1" ]; then echo $lineCount; return; fi + # Filtering out commented and empty lines + let lineCount=`cat $1 | grep -v '#' | grep -v ^$ | wc -l` + echo $lineCount + return +} + # convert relative path to absolute path bin=`dirname "$this"` script=`basename "$this"` @@ -180,18 +251,61 @@ else fi fi -# Default memory settings if none provided by the environment or +# Execute distrib-setup.sh for any distribution-specific setup (e.g. checks). +# distrib-setup.sh is optional; it is created by some distribution installers +# that need additional distribution-specific setup to be done. +# Because installers will have site-specific steps, the file +# should be moved into the site directory, if the user employs one. + +# Checking if being executed in context of Drillbit and not SQLLine +if [ "$DRILLBIT_CONTEXT" == "1" ]; then + # Check whether to run exclusively distrib-setup.sh OR auto-setup.sh + distribSetup="$DRILL_CONF_DIR/distrib-setup.sh" ; #Site-based distrib-setup.sh + if [ $(checkExecutableLineCount $distribSetup) -eq 0 ]; then + distribSetup="$DRILL_HOME/conf/distrib-setup.sh" ; #Install-based distrib-setup.sh + if [ $(checkExecutableLineCount $distribSetup) -eq 0 ]; then + # Run Default Auto Setup + distribSetup="$DRILL_HOME/bin/auto-setup.sh" + fi + fi + # Check and run additional setup defined by user + drillSetup="$DRILL_CONF_DIR/drill-setup.sh" ; #Site-based drill-setup.sh + if [ $(checkExecutableLineCount $drillSetup) -eq 0 ]; then + drillSetup="$DRILL_HOME/conf/drill-setup.sh" ; #Install-based drill-setup.sh + if [ $(checkExecutableLineCount $drillSetup) -eq 0 ]; then drillSetup=""; fi + fi + + # Enforcing checks in order (distrib-setup.sh , drill-setup.sh) + # (NOTE: A script is executed only if it has relevant executable lines) + # Both distribSetup & drillSetup are executed because the user might have introduced additional checks + if [ -n "$distribSetup" ]; then + . "$distribSetup" + if [ $? -gt 0 ]; then fatal_error "Aborting Drill Startup due failed setup by $distribSetup"; fi + fi + if [ -n "$drillSetup" ] && [ $(checkExecutableLineCount $drillSetup) -gt 0 ]; then + . "$drillSetup" + if [ $? -gt 0 ]; then fatal_error "Aborting Drill Startup due failed setup from $drillSetup"; fi + fi +fi + +# DEFAULT memory settings if none provided by the environment or # above config files. # The Drillbit needs a large code cache. - export DRILL_MAX_DIRECT_MEMORY=${DRILL_MAX_DIRECT_MEMORY:-"8G"} export DRILL_HEAP=${DRILL_HEAP:-"4G"} -export DRILLBIT_MAX_PERM=${DRILLBIT_MAX_PERM:-"512M"} export DRILLBIT_CODE_CACHE_SIZE=${DRILLBIT_CODE_CACHE_SIZE:-"1G"} +export DRILLBIT_MAX_PERM=${DRILLBIT_MAX_PERM:-"512M"} export DRILLBIT_OPTS="-Xms$DRILL_HEAP -Xmx$DRILL_HEAP -XX:MaxDirectMemorySize=$DRILL_MAX_DIRECT_MEMORY" export DRILLBIT_OPTS="$DRILLBIT_OPTS -XX:ReservedCodeCacheSize=$DRILLBIT_CODE_CACHE_SIZE -Ddrill.exec.enable-epoll=false" -export DRILLBIT_OPTS="$DRILLBIT_OPTS -XX:MaxPermSize=$DRILLBIT_MAX_PERM" + +# Remove MaxPermSize for JVM version >= 1.8.0 +jvmVersion=`"$JAVA" -version 2>&1 | grep "version" | tr '"_' '\n' | grep '[0-9].[0.9]'` +if [[ $jvmVersion < "1.8.0" ]]; then + export DRILLBIT_OPTS="$DRILLBIT_OPTS -XX:MaxPermSize=$DRILLBIT_MAX_PERM" +else + unset DRILLBIT_MAX_PERM +fi # Under YARN, the log directory is usually YARN-provided. Replace any # value that may have been set in drill-env.sh. @@ -364,62 +478,6 @@ if [[ ! -d "$DRILL_TMP_DIR" || ! -w "$DRILL_TMP_DIR" ]]; then fatal_error "Temporary directory does not exist or is not writable: $DRILL_TMP_DIR" fi -# Test for cygwin -is_cygwin=false -case "`uname`" in -CYGWIN*) is_cygwin=true;; -esac - -if $is_cygwin; then - JAVA_BIN="java.exe" -else - JAVA_BIN="java" -fi - -# Test for or find JAVA_HOME - -if [ -z "$JAVA_HOME" ]; then - SOURCE=`which java` - if [ -e $SOURCE ]; then - while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - SOURCE="$(readlink "$SOURCE")" - # if $SOURCE was a relative symlink, we need to resolve it relative - # to the path where the symlink file was located - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" - done - JAVA_HOME="$( cd -P "$( dirname "$SOURCE" )" && cd .. && pwd )" - JAVA=$SOURCE - fi - # if we didn't set it - if [ -z "$JAVA_HOME" ]; then - fatal_error "JAVA_HOME is not set and Java could not be found" - fi -fi - -# Now, verify that 'java' binary exists and is suitable for Drill. -# If we started with `which java` above, use that path (after replacing -# symlinks.) If we started with JAVA_HOME, try in bin. Doing so handles -# the case in which JAVA_HOME is a JDK that has a nested JRE; we prefer -# the JDK bin. Finally, if nothing else works, just search for the -# executable. - -if [ -z "$JAVA" ]; then - JAVA="$JAVA_HOME/bin/$JAVA_BIN" - if [[ ! -e $JAVA ]]; then - JAVA=`find -L "$JAVA_HOME" -name $JAVA_BIN -type f | head -n 1` - fi -fi -if [ ! -e "$JAVA" ]; then - fatal_error "Java not found at JAVA_HOME=$JAVA_HOME." -fi - -# Ensure that Java version is at least 1.7 -"$JAVA" -version 2>&1 | grep "version" | egrep -e "1\.4|1\.5|1\.6" > /dev/null -if [ $? -eq 0 ]; then - fatal_error "Java 1.7 or later is required to run Apache Drill." -fi - # Adjust paths for CYGWIN if $is_cygwin; then DRILL_HOME=`cygpath -w "$DRILL_HOME"` http://git-wip-us.apache.org/repos/asf/drill/blob/266250cf/distribution/src/resources/drill-setup.sh ---------------------------------------------------------------------- diff --git a/distribution/src/resources/drill-setup.sh b/distribution/src/resources/drill-setup.sh new file mode 100644 index 0000000..f899b65 --- /dev/null +++ b/distribution/src/resources/drill-setup.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is empty by default. It provides user-specific startup checks and +# is invoked by drill-config.sh after any distribution-specific checks (and +# setting of environment variables) have been performed. +# To FAIL any check, return with a non-zero return code +# e.g. +# if [ $status == "FAILED" ]; return 1; fi + +###========================================================================== +# FEATURES (Added here to describe supported additions) +###==========================================================================