Author: acmurthy Date: Tue Sep 25 23:49:09 2012 New Revision: 1390224 URL: http://svn.apache.org/viewvc?rev=1390224&view=rev Log: MAPREDUCE-4649. Ensure MapReduce JobHistory Daemon doens't assume HADOOP_YARN_HOME and HADOOP_MAPRED_HOME are the same. Contributed by Vinod K V.
Added: hadoop/common/trunk/hadoop-mapreduce-project/conf/mapred-env.sh Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt hadoop/common/trunk/hadoop-mapreduce-project/INSTALL hadoop/common/trunk/hadoop-mapreduce-project/bin/mapred hadoop/common/trunk/hadoop-mapreduce-project/bin/mapred-config.sh hadoop/common/trunk/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1390224&r1=1390223&r2=1390224&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Sep 25 23:49:09 2012 @@ -339,6 +339,10 @@ Release 2.0.2-alpha - 2012-09-07 MAPREDUCE-4380. Empty Userlogs directory is getting created under logs directory (Devaraj K via bobby) + MAPREDUCE-4649. Ensure MapReduce JobHistory Daemon doens't assume + HADOOP_YARN_HOME and HADOOP_MAPRED_HOME are the same. (vinodkv via + acmurthy) + Release 2.0.0-alpha - 05-23-2012 INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-mapreduce-project/INSTALL URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/INSTALL?rev=1390224&r1=1390223&r2=1390224&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/INSTALL (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/INSTALL Tue Sep 25 23:49:09 2012 @@ -19,7 +19,7 @@ mvn clean package -Pdist -Dtar -DskipTes You can omit -Pnative it you don't want to build native packages. Step 4) Untar the tarball from hadoop-dist/target/ into a clean and different -directory, say YARN_HOME. +directory, say HADOOP_YARN_HOME. Step 5) Start hdfs @@ -32,7 +32,7 @@ You probably want to export these in had export HADOOP_MAPRED_HOME=<mapred loc> export HADOOP_COMMON_HOME=<common loc> export HADOOP_HDFS_HOME=<hdfs loc> -export YARN_HOME=directory where you untarred yarn +export HADOOP_YARN_HOME=directory where you untarred yarn export HADOOP_CONF_DIR=<conf loc> export YARN_CONF_DIR=$HADOOP_CONF_DIR @@ -53,7 +53,7 @@ Step 8) Modify mapred-site.xml to use ya <value>yarn</value> </property> -Step 9) cd $YARN_HOME +Step 9) cd $HADOOP_YARN_HOME Step 10) sbin/yarn-daemon.sh start resourcemanager @@ -64,7 +64,7 @@ Step 12) sbin/mr-jobhistory-daemon.sh st Step 13) You are all set, an example on how to run a mapreduce job is: cd $HADOOP_MAPRED_HOME ant examples -Dresolvers=internal -$HADOOP_COMMON_HOME/bin/hadoop jar $HADOOP_MAPRED_HOME/build/hadoop-mapreduce-examples-*.jar randomwriter -Dmapreduce.job.user.name=$USER -Dmapreduce.randomwriter.bytespermap=10000 -Ddfs.blocksize=536870912 -Ddfs.block.size=536870912 -libjars $YARN_HOME/modules/hadoop-mapreduce-client-jobclient-*.jar output +$HADOOP_COMMON_HOME/bin/hadoop jar $HADOOP_MAPRED_HOME/build/hadoop-mapreduce-examples-*.jar randomwriter -Dmapreduce.job.user.name=$USER -Dmapreduce.randomwriter.bytespermap=10000 -Ddfs.blocksize=536870912 -Ddfs.block.size=536870912 -libjars $HADOOP_YARN_HOME/modules/hadoop-mapreduce-client-jobclient-*.jar output The output on the command line should be almost similar to what you see in the JT/TT setup (Hadoop 0.20/0.21) Modified: hadoop/common/trunk/hadoop-mapreduce-project/bin/mapred URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/bin/mapred?rev=1390224&r1=1390223&r2=1390224&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/bin/mapred (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/bin/mapred Tue Sep 25 23:49:09 2012 @@ -115,6 +115,11 @@ for f in $HADOOP_MAPRED_HOME/${MAPRED_DI CLASSPATH=${CLASSPATH}:$f; done +# Need YARN jars also +for f in $HADOOP_YARN_HOME/${YARN_DIR}/*.jar; do + CLASSPATH=${CLASSPATH}:$f; +done + # add libs to CLASSPATH for f in $HADOOP_MAPRED_HOME/${MAPRED_LIB_JARS_DIR}/*.jar; do CLASSPATH=${CLASSPATH}:$f; Modified: hadoop/common/trunk/hadoop-mapreduce-project/bin/mapred-config.sh URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/bin/mapred-config.sh?rev=1390224&r1=1390223&r2=1390224&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/bin/mapred-config.sh (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/bin/mapred-config.sh Tue Sep 25 23:49:09 2012 @@ -38,3 +38,14 @@ else echo "Hadoop common not found." exit fi + +# some more specific variables +export HADOOP_MAPRED_LOG_DIR=${HADOOP_MAPRED_LOG_DIR:-${HADOOP_MAPRED_HOME}/logs} +export HADOOP_MAPRED_LOGFILE=${HADOOP_MAPRED_LOGFILE:-hadoop.log} + +HADOOP_MAPRED_ROOT_LOGGER=${HADOOP_MAPRED_ROOT_LOGGER:-INFO,console} +HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_MAPRED_LOG_DIR" +HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_MAPRED_LOGFILE" +export HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.root.logger=${HADOOP_MAPRED_ROOT_LOGGER}" + + Modified: hadoop/common/trunk/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh?rev=1390224&r1=1390223&r2=1390224&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh Tue Sep 25 23:49:09 2012 @@ -16,22 +16,16 @@ # limitations under the License. -# Runs a yarn command as a daemon. # # Environment Variables # -# HADOOP_LOGFILE Hadoop log file. -# HADOOP_ROOT_LOGGER Hadoop root logger. # HADOOP_JHS_LOGGER Hadoop JobSummary logger. -# YARN_CONF_DIR Alternate conf dir. Default is ${YARN_HOME}/conf. -# YARN_LOG_DIR Where log files are stored. PWD by default. -# YARN_MASTER host:path where hadoop code should be rsync'd from -# YARN_PID_DIR The pid files are stored. /tmp by default. -# YARN_IDENT_STRING A string representing this instance of hadoop. $USER by default -# YARN_NICENESS The scheduling priority for daemons. Defaults to 0. +# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_MAPRED_HOME}/conf. +# HADOOP_MAPRED_PID_DIR The pid files are stored. /tmp by default. +# HADOOP_MAPRED_NICENESS The scheduling priority for daemons. Defaults to 0. ## -usage="Usage: mr-jobhistory-daemon.sh [--config <conf-dir>] [--hosts hostlistfile] (start|stop) <mapred-command> " +usage="Usage: mr-jobhistory-daemon.sh [--config <conf-dir>] (start|stop) <mapred-command> " # if no args specified, show usage if [ $# -le 1 ]; then @@ -42,10 +36,6 @@ fi bin=`dirname "${BASH_SOURCE-$0}"` bin=`cd "$bin"; pwd` -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -. $HADOOP_LIBEXEC_DIR/yarn-config.sh - # get arguments startStop=$1 shift @@ -69,43 +59,47 @@ hadoop_rotate_log () fi } -if [ -f "${YARN_CONF_DIR}/yarn-env.sh" ]; then - . "${YARN_CONF_DIR}/yarn-env.sh" +if [ "$HADOOP_MAPRED_IDENT_STRING" = "" ]; then + export HADOOP_MAPRED_IDENT_STRING="$USER" fi -if [ "$YARN_IDENT_STRING" = "" ]; then - export YARN_IDENT_STRING="$USER" -fi +export HADOOP_MAPRED_HOME=${HADOOP_MAPRED_HOME:-${HADOOP_PREFIX}} +export HADOOP_MAPRED_LOGFILE=mapred-$HADOOP_MAPRED_IDENT_STRING-$command-$HOSTNAME.log +export HADOOP_MAPRED_ROOT_LOGGER=${HADOOP_MAPRED_ROOT_LOGGER:-INFO,RFA} +export HADOOP_JHS_LOGGER=${HADOOP_JHS_LOGGER:-INFO,JSA} + +DEFAULT_LIBEXEC_DIR="$bin"/../libexec +HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} +. $HADOOP_LIBEXEC_DIR/mapred-config.sh -# get log directory -if [ "$YARN_LOG_DIR" = "" ]; then - export YARN_LOG_DIR="$YARN_HOME/logs" +if [ -f "${HADOOP_CONF_DIR}/mapred-env.sh" ]; then + . "${HADOOP_CONF_DIR}/mapred-env.sh" fi -mkdir -p "$YARN_LOG_DIR" -chown $YARN_IDENT_STRING $YARN_LOG_DIR -if [ "$YARN_PID_DIR" = "" ]; then - YARN_PID_DIR=/tmp +mkdir -p "$HADOOP_MAPRED_LOG_DIR" +chown $HADOOP_MAPRED_IDENT_STRING $HADOOP_MAPRED_LOG_DIR + +if [ "$HADOOP_MAPRED_PID_DIR" = "" ]; then + HADOOP_MAPRED_PID_DIR=/tmp fi -# some variables -export HADOOP_LOGFILE=yarn-$YARN_IDENT_STRING-$command-$HOSTNAME.log -export HADOOP_ROOT_LOGGER=${HADOOP_ROOT_LOGGER:-INFO,RFA} -export HADOOP_JHS_LOGGER=${HADOOP_JHS_LOGGER:-INFO,JSA} -log=$YARN_LOG_DIR/yarn-$YARN_IDENT_STRING-$command-$HOSTNAME.out -pid=$YARN_PID_DIR/yarn-$YARN_IDENT_STRING-$command.pid -YARN_STOP_TIMEOUT=${YARN_STOP_TIMEOUT:-5} +HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.id.str=$HADOOP_MAPRED_IDENT_STRING" + +log=$HADOOP_MAPRED_LOG_DIR/mapred-$HADOOP_MAPRED_IDENT_STRING-$command-$HOSTNAME.out +pid=$HADOOP_MAPRED_PID_DIR/mapred-$HADOOP_MAPRED_IDENT_STRING-$command.pid + +HADOOP_MAPRED_STOP_TIMEOUT=${HADOOP_MAPRED_STOP_TIMEOUT:-5} # Set default scheduling priority -if [ "$YARN_NICENESS" = "" ]; then - export YARN_NICENESS=0 +if [ "$HADOOP_MAPRED_NICENESS" = "" ]; then + export HADOOP_MAPRED_NICENESS=0 fi case $startStop in (start) - mkdir -p "$YARN_PID_DIR" + mkdir -p "$HADOOP_MAPRED_PID_DIR" if [ -f $pid ]; then if kill -0 `cat $pid` > /dev/null 2>&1; then @@ -114,15 +108,10 @@ case $startStop in fi fi - if [ "$YARN_MASTER" != "" ]; then - echo rsync from $YARN_MASTER - rsync -a -e ssh --delete --exclude=.svn --exclude='logs/*' --exclude='contrib/hod/logs/*' $YARN_MASTER/ "$YARN_HOME" - fi - hadoop_rotate_log $log echo starting $command, logging to $log - cd "$YARN_HOME" - nohup nice -n $YARN_NICENESS "$YARN_HOME"/bin/mapred --config $YARN_CONF_DIR $command "$@" > "$log" 2>&1 < /dev/null & + cd "$HADOOP_MAPRED_HOME" + nohup nice -n $HADOOP_MAPRED_NICENESS "$HADOOP_MAPRED_HOME"/bin/mapred --config $HADOOP_CONF_DIR $command "$@" > "$log" 2>&1 < /dev/null & echo $! > $pid sleep 1; head "$log" ;; @@ -134,9 +123,9 @@ case $startStop in if kill -0 $TARGET_PID > /dev/null 2>&1; then echo stopping $command kill $TARGET_PID - sleep $YARN_STOP_TIMEOUT + sleep $HADOOP_MAPRED_STOP_TIMEOUT if kill -0 $TARGET_PID > /dev/null 2>&1; then - echo "$command did not stop gracefully after $YARN_STOP_TIMEOUT seconds: killing with kill -9" + echo "$command did not stop gracefully after $HADOOP_MAPRED_STOP_TIMEOUT seconds: killing with kill -9" kill -9 $TARGET_PID fi else Added: hadoop/common/trunk/hadoop-mapreduce-project/conf/mapred-env.sh URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/conf/mapred-env.sh?rev=1390224&view=auto ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/conf/mapred-env.sh (added) +++ hadoop/common/trunk/hadoop-mapreduce-project/conf/mapred-env.sh Tue Sep 25 23:49:09 2012 @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# export JAVA_HOME=/home/y/libexec/jdk1.6.0/ + +export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=1000 + +export HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA + +#export HADOOP_JOB_HISTORYSERVER_OPTS= +#export HADOOP_MAPRED_LOG_DIR="" # Where log files are stored. $HADOOP_MAPRED_HOME/logs by default. +#export HADOOP_JHS_LOGGER=INFO,RFA # Hadoop JobSummary logger. +#export HADOOP_MAPRED_PID_DIR= # The pid files are stored. /tmp by default. +#export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default +#export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0. Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml?rev=1390224&r1=1390223&r2=1390224&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml Tue Sep 25 23:49:09 2012 @@ -102,7 +102,7 @@ <property><!--Loaded from job.xml--><name>dfs.permissions.enabled</name><value>true</value></property> <property><!--Loaded from job.xml--><name>mapreduce.tasktracker.taskcontroller</name><value>org.apache.hadoop.mapred.DefaultTaskController</value></property> <property><!--Loaded from job.xml--><name>mapreduce.reduce.shuffle.parallelcopies</name><value>5</value></property> -<property><!--Loaded from job.xml--><name>yarn.nodemanager.env-whitelist</name><value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,YARN_HOME</value></property> +<property><!--Loaded from job.xml--><name>yarn.nodemanager.env-whitelist</name><value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,HADOOP_YARN_HOME</value></property> <property><!--Loaded from job.xml--><name>mapreduce.jobtracker.heartbeats.in.second</name><value>100</value></property> <property><!--Loaded from job.xml--><name>mapreduce.job.maxtaskfailures.per.tracker</name><value>4</value></property> <property><!--Loaded from job.xml--><name>ipc.client.connection.maxidletime</name><value>10000</value></property> @@ -317,8 +317,8 @@ $HADOOP_COMMON_HOME/share/hadoop/common/lib/*, $HADOOP_HDFS_HOME/share/hadoop/hdfs/*, $HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*, - $YARN_HOME/share/hadoop/mapreduce/*, - $YARN_HOME/share/hadoop/mapreduce/lib/* + $HADOOP_YARN_HOME/share/hadoop/mapreduce/*, + $HADOOP_YARN_HOME/share/hadoop/mapreduce/lib/* </value></property> <property><!--Loaded from job.xml--><name>yarn.nodemanager.log-aggregation.compression-type</name><value>gz</value></property> <property><!--Loaded from job.xml--><name>dfs.image.compress</name><value>false</value></property>