Repository: drill Updated Branches: refs/heads/master da241134f -> f8691f4f9
DRILL-143: Support CGROUPs resource management Introduces the DRILLBIT_CGROUP option in drill-env.sh. The startup script checks if the specified CGroup (ver 2) is available and tries to apply it to the launched Drillbit JVM. This would benefit not just Drill-on-YARN usecases, but any setup that would like CGroups for enforcement of (cpu) resources management. (Also introduced SYS_CGROUP_DIR to account for possible non default locations of CGroup). e.g when Drillbit is configured to use `drillcpu` cgroup ``` [root@maprlabs ~]# /opt/mapr/drill/apache-drill-1.14.0-SNAPSHOT/bin/drillbit.sh restart Stopping drillbit .. Starting drillbit, logging to /var/log/drill/drillbit.out WARN: Drillbit's CPU resource usage will be managed under the CGroup : drillcpu (up to 4.00 cores allowed) ``` e.g. Non-existent CGroup `droolcpu` is used ``` [root@kk127 ~]# /opt/mapr/drill/apache-drill-1.14.0-SNAPSHOT/bin/drillbit.sh restart Stopping drillbit .. Starting drillbit, logging to /var/log/drill/drillbit.out ERROR: cgroup droolcpu does not found. Ensure that daemon is running and cgroup exists ``` closes #1200 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/6370ed63 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/6370ed63 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/6370ed63 Branch: refs/heads/master Commit: 6370ed630b6bc56972d8893618773b2176846085 Parents: da24113 Author: Kunal Khatua <kkha...@maprtech.com> Authored: Mon Apr 2 22:35:53 2018 -0700 Committer: Ben-Zvi <bben-...@mapr.com> Committed: Fri Apr 13 14:56:49 2018 -0700 ---------------------------------------------------------------------- distribution/src/resources/drill-env.sh | 10 +++++++ distribution/src/resources/drillbit.sh | 39 ++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/6370ed63/distribution/src/resources/drill-env.sh ---------------------------------------------------------------------- diff --git a/distribution/src/resources/drill-env.sh b/distribution/src/resources/drill-env.sh index 351d867..4dd52aa 100755 --- a/distribution/src/resources/drill-env.sh +++ b/distribution/src/resources/drill-env.sh @@ -86,6 +86,16 @@ #export DRILL_PID_DIR=${DRILL_PID_DIR:-$DRILL_HOME} +# Default (Standard) CGroup Location: /sys/fs/cgroup +# Specify the cgroup location if it is different from the default +#export SYS_CGROUP_DIR=${SYS_CGROUP_DIR:-"/sys/fs/cgroup"} + +# CGroup to which the Drillbit belongs when running as a daemon using drillbit.sh start . +# Drill will use CGroup for CPU enforcement only. + +# Unset $DRILLBIT_CGROUP by default +#export DRILLBIT_CGROUP=${DRILLBIT_CGROUP:-"drillcpu"} + # Custom JVM arguments to pass to the both the Drillbit and sqlline. Typically # used to override system properties as shown below. Empty by default. http://git-wip-us.apache.org/repos/asf/drill/blob/6370ed63/distribution/src/resources/drillbit.sh ---------------------------------------------------------------------- diff --git a/distribution/src/resources/drillbit.sh b/distribution/src/resources/drillbit.sh index 11a124e..3634753 100755 --- a/distribution/src/resources/drillbit.sh +++ b/distribution/src/resources/drillbit.sh @@ -127,6 +127,42 @@ check_before_start() fi } +check_after_start(){ + dbitPid=$1; + # Check and enforce for CGroup + if [ -n "$DRILLBIT_CGROUP" ]; then + check_and_enforce_cgroup $dbitPid + fi +} + +check_and_enforce_cgroup(){ + dbitPid=$1; + kill -0 $dbitPid + if [ $? -gt 0 ]; then + echo "ERROR: Failed to add Drillbit to CGroup ( $DRILLBIT_CGROUP ) for 'cpu'. Ensure that the Drillbit ( pid=$dbitPid ) started up." >&2 + exit 1 + fi + SYS_CGROUP_DIR=${SYS_CGROUP_DIR:-"/sys/fs/cgroup"} + if [ -f $SYS_CGROUP_DIR/cpu/$DRILLBIT_CGROUP/cgroup.procs ]; then + echo $dbitPid > $SYS_CGROUP_DIR/cpu/$DRILLBIT_CGROUP/cgroup.procs + # Verify Enforcement + cgroupStatus=`grep -w $pid $SYS_CGROUP_DIR/cpu/${DRILLBIT_CGROUP}/cgroup.procs` + if [ -z "$cgroupStatus" ]; then + #Ref: https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt + cpu_quota=`cat ${SYS_CGROUP_DIR}/cpu/${DRILLBIT_CGROUP}/cpu.cfs_quota_us` + cpu_period=`cat ${SYS_CGROUP_DIR}/cpu/${DRILLBIT_CGROUP}/cpu.cfs_period_us` + if [ $cpu_period -gt 0 ] && [ $cpu_quota -gt 0 ]; then + coresAllowed=`echo $(( 100 * $cpu_quota / $cpu_period )) | sed 's/..$/.&/'` + echo "INFO: CGroup (drillcpu) will limit Drill to $coresAllowed cpu(s)" + fi + else + echo "ERROR: Failed to add Drillbit to CGroup ( $DRILLBIT_CGROUP ) for 'cpu'. Ensure that the cgroup manages 'cpu'" >&2 + fi + else + echo "ERROR: CGroup $DRILLBIT_CGROUP not found. Ensure that daemon is running, SYS_CGROUP_DIR is correctly set (currently, $SYS_CGROUP_DIR ), and that the CGroup exists" >&2 + fi +} + wait_until_done () { p=$1 @@ -152,8 +188,11 @@ start_bit ( ) echo "`date` Starting $command on `hostname`" >> "$DRILLBIT_LOG_PATH" echo "`ulimit -a`" >> "$DRILLBIT_LOG_PATH" 2>&1 nohup nice -n $DRILL_NICENESS "$DRILL_HOME/bin/runbit" exec ${args[@]} >> "$logout" 2>&1 & + procId=$! + echo $procId > $pid # Yeah, $pid is a file, $procId is the pid... echo $! > $pid sleep 1 + check_after_start $procId } stop_bit ( )
