Author: wang Date: Wed Aug 20 01:22:42 2014 New Revision: 1619018 URL: http://svn.apache.org/r1619018 Log: Merge from trunk to branch.
Modified: hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/ (props changed) hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/CHANGES.txt (contents, props changed) hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/bin/mapred hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/bin/mapred-config.sh hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/conf/ (props changed) hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/conf/mapred-env.sh hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/OracleDBRecordReader.java hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestDbClasses.java Propchange: hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1618694-1618997 Modified: hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/CHANGES.txt?rev=1619018&r1=1619017&r2=1619018&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/CHANGES.txt Wed Aug 20 01:22:42 2014 @@ -233,6 +233,9 @@ Release 2.6.0 - UNRELEASED MAPREDUCE-6036. TestJobEndNotifier fails intermittently in branch-2 (chang li via jlowe) + MAPREDUCE-6012. DBInputSplit creates invalid ranges on Oracle. + (Wei Yan via kasha) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES Propchange: hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/CHANGES.txt ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1618694-1618997 Modified: hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/bin/mapred URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/bin/mapred?rev=1619018&r1=1619017&r2=1619018&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/bin/mapred (original) +++ hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/bin/mapred Wed Aug 20 01:22:42 2014 @@ -15,138 +15,129 @@ # See the License for the specific language governing permissions and # limitations under the License. -bin=`which $0` -bin=`dirname ${bin}` -bin=`cd "$bin" > /dev/null; pwd` - -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -if [ -e ${HADOOP_LIBEXEC_DIR}/mapred-config.sh ]; then - . ${HADOOP_LIBEXEC_DIR}/mapred-config.sh -else - . "$bin/mapred-config.sh" -fi - -function print_usage(){ - echo "Usage: mapred [--config confdir] COMMAND" +function hadoop_usage +{ + echo "Usage: mapred [--config confdir] [--daemon (start|stop|status)] COMMAND" echo " where COMMAND is one of:" - echo " pipes run a Pipes job" - echo " job manipulate MapReduce jobs" - echo " queue get information regarding JobQueues" + + echo " archive -archiveName NAME -p <parent path> <src>* <dest> create a hadoop archive" echo " classpath prints the class path needed for running" echo " mapreduce subcommands" - echo " historyserver run job history servers as a standalone daemon" echo " distcp <srcurl> <desturl> copy file or directories recursively" - echo " archive -archiveName NAME -p <parent path> <src>* <dest> create a hadoop archive" - echo " hsadmin job history server admin interface" + echo " job manipulate MapReduce jobs" + echo " historyserver run job history servers as a standalone daemon" + echo " pipes run a Pipes job" + echo " queue get information regarding JobQueues" + echo " sampler sampler" echo "" echo "Most commands print help when invoked w/o parameters." } +this="${BASH_SOURCE-$0}" +bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) + +# let's locate libexec... +if [[ -n "${HADOOP_PREFIX}" ]]; then + DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +else + DEFAULT_LIBEXEC_DIR="${bin}/../libexec" +fi + +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/mapred-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/mapred-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/mapred-config.sh." 2>&1 + exit 1 +fi + + if [ $# = 0 ]; then - print_usage - exit + hadoop_exit_with_usage 1 fi COMMAND=$1 shift -case $COMMAND in - # usage flags - --help|-help|-h) - print_usage - exit - ;; +case ${COMMAND} in + mradmin|jobtracker|tasktracker|groups) + echo "Sorry, the ${COMMAND} command is no longer supported." + echo "You may find similar functionality with the \"yarn\" shell command." + hadoop_exit_with_usage 1 + ;; + archive) + CLASS=org.apache.hadoop.tools.HadoopArchives + hadoop_add_classpath "${TOOL_PATH}" + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" + ;; + classpath) + hadoop_finalize + echo "${CLASSPATH}" + exit 0 + ;; + distcp) + CLASS=org.apache.hadoop.tools.DistCp + hadoop_add_classpath "${TOOL_PATH}" + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" + ;; + historyserver) + daemon="true" + CLASS=org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_JOB_HISTORYSERVER_OPTS}" + if [ -n "${HADOOP_JOB_HISTORYSERVER_HEAPSIZE}" ]; then + JAVA_HEAP_MAX="-Xmx${HADOOP_JOB_HISTORYSERVER_HEAPSIZE}m" + fi + HADOOP_DAEMON_ROOT_LOGGER=${HADOOP_JHS_LOGGER:-$HADOOP_DAEMON_ROOT_LOGGER} + ;; + job) + CLASS=org.apache.hadoop.mapred.JobClient + ;; + pipes) + CLASS=org.apache.hadoop.mapred.pipes.Submitter + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" + ;; + queue) + CLASS=org.apache.hadoop.mapred.JobQueueClient + ;; + sampler) + CLASS=org.apache.hadoop.mapred.lib.InputSampler + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" + ;; + -*|*) + hadoop_exit_with_usage 1 + ;; esac -if [ "$COMMAND" = "job" ] ; then - CLASS=org.apache.hadoop.mapred.JobClient - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" -elif [ "$COMMAND" = "queue" ] ; then - CLASS=org.apache.hadoop.mapred.JobQueueClient - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" -elif [ "$COMMAND" = "pipes" ] ; then - CLASS=org.apache.hadoop.mapred.pipes.Submitter - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" -elif [ "$COMMAND" = "sampler" ] ; then - CLASS=org.apache.hadoop.mapred.lib.InputSampler - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" -elif [ "$COMMAND" = "classpath" ] ; then - echo -n -elif [ "$COMMAND" = "historyserver" ] ; then - CLASS=org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer - HADOOP_OPTS="$HADOOP_OPTS -Dmapred.jobsummary.logger=${HADOOP_JHS_LOGGER:-INFO,console} $HADOOP_JOB_HISTORYSERVER_OPTS" - if [ "$HADOOP_JOB_HISTORYSERVER_HEAPSIZE" != "" ]; then - JAVA_HEAP_MAX="-Xmx""$HADOOP_JOB_HISTORYSERVER_HEAPSIZE""m" - fi -elif [ "$COMMAND" = "mradmin" ] \ - || [ "$COMMAND" = "jobtracker" ] \ - || [ "$COMMAND" = "tasktracker" ] \ - || [ "$COMMAND" = "groups" ] ; then - echo "Sorry, the $COMMAND command is no longer supported." - echo "You may find similar functionality with the \"yarn\" shell command." - print_usage - exit 1 -elif [ "$COMMAND" = "distcp" ] ; then - CLASS=org.apache.hadoop.tools.DistCp - CLASSPATH=${CLASSPATH}:${TOOL_PATH} - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" -elif [ "$COMMAND" = "archive" ] ; then - CLASS=org.apache.hadoop.tools.HadoopArchives - CLASSPATH=${CLASSPATH}:${TOOL_PATH} - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" -elif [ "$COMMAND" = "hsadmin" ] ; then - CLASS=org.apache.hadoop.mapreduce.v2.hs.client.HSAdmin - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" -else - echo $COMMAND - invalid command - print_usage - exit 1 -fi +daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out" +daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid" -# for developers, add mapred classes to CLASSPATH -if [ -d "$HADOOP_MAPRED_HOME/build/classes" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/classes -fi -if [ -d "$HADOOP_MAPRED_HOME/build/webapps" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build -fi -if [ -d "$HADOOP_MAPRED_HOME/build/test/classes" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/test/classes -fi -if [ -d "$HADOOP_MAPRED_HOME/build/tools" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/tools -fi -# for releases, add core mapred jar & webapps to CLASSPATH -if [ -d "$HADOOP_PREFIX/${MAPRED_DIR}/webapps" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/${MAPRED_DIR} -fi -for f in $HADOOP_MAPRED_HOME/${MAPRED_DIR}/*.jar; do - CLASSPATH=${CLASSPATH}:$f; -done - -# Need YARN jars also -for f in $HADOOP_YARN_HOME/${YARN_DIR}/*.jar; do - CLASSPATH=${CLASSPATH}:$f; -done - -# add libs to CLASSPATH -for f in $HADOOP_MAPRED_HOME/${MAPRED_LIB_JARS_DIR}/*.jar; do - CLASSPATH=${CLASSPATH}:$f; -done - -# add modules to CLASSPATH -for f in $HADOOP_MAPRED_HOME/modules/*.jar; do - CLASSPATH=${CLASSPATH}:$f; -done - -if [ "$COMMAND" = "classpath" ] ; then - echo $CLASSPATH - exit +if [[ "${HADOOP_DAEMON_MODE}" != "default" ]]; then + # shellcheck disable=SC2034 + HADOOP_ROOT_LOGGER="${HADOOP_DAEMON_ROOT_LOGGER}" + hadoop_add_param HADOOP_OPTS mapred.jobsummary.logger "-Dmapred.jobsummary.logger=${HADOOP_ROOT_LOGGER}" + # shellcheck disable=SC2034 + HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.log" fi -HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,NullAppender}" +hadoop_add_param HADOOP_OPTS Xmx "${JAVA_HEAP_MAX}" +hadoop_finalize export CLASSPATH -exec "$JAVA" -Dproc_$COMMAND $JAVA_HEAP_MAX $HADOOP_OPTS $CLASS "$@" + +if [[ -n "${daemon}" ]]; then + if [[ -n "${secure_service}" ]]; then + hadoop_secure_daemon_handler "${HADOOP_DAEMON_MODE}" "${COMMAND}"\ + "${CLASS}" "${daemon_pidfile}" "${daemon_outfile}" \ + "${priv_pidfile}" "${priv_outfile}" "${priv_errfile}" "$@" + else + hadoop_daemon_handler "${HADOOP_DAEMON_MODE}" "${COMMAND}" "${CLASS}" \ + "${daemon_pidfile}" "${daemon_outfile}" "$@" + fi + exit $? +else + hadoop_java_exec "${COMMAND}" "${CLASS}" "$@" +fi + Modified: hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/bin/mapred-config.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/bin/mapred-config.sh?rev=1619018&r1=1619017&r2=1619018&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/bin/mapred-config.sh (original) +++ hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/bin/mapred-config.sh Wed Aug 20 01:22:42 2014 @@ -18,35 +18,55 @@ # included in all the mapred scripts with source command # should not be executed directly -bin=`which "$0"` -bin=`dirname "${bin}"` -bin=`cd "$bin"; pwd` +function hadoop_subproject_init +{ + if [ -e "${HADOOP_CONF_DIR}/mapred-env.sh" ]; then + . "${HADOOP_CONF_DIR}/mapred-env.sh" + fi + + # at some point in time, someone thought it would be a good idea to + # create separate vars for every subproject. *sigh* + # let's perform some overrides and setup some defaults for bw compat + # this way the common hadoop var's == subproject vars and can be + # used interchangeable from here on out + # ... + # this should get deprecated at some point. + HADOOP_LOG_DIR="${HADOOP_MAPRED_LOG_DIR:-$HADOOP_LOG_DIR}" + HADOOP_MAPRED_LOG_DIR="${HADOOP_LOG_DIR}" + + HADOOP_LOGFILE="${HADOOP_MAPRED_LOGFILE:-$HADOOP_LOGFILE}" + HADOOP_MAPRED_LOGFILE="${HADOOP_LOGFILE}" + + HADOOP_NICENESS="${HADOOP_MAPRED_NICENESS:-$HADOOP_NICENESS}" + HADOOP_MAPRED_NICENESS="${HADOOP_NICENESS}" + + HADOOP_STOP_TIMEOUT="${HADOOP_MAPRED_STOP_TIMEOUT:-$HADOOP_STOP_TIMEOUT}" + HADOOP_MAPRED_STOP_TIMEOUT="${HADOOP_STOP_TIMEOUT}" + + HADOOP_PID_DIR="${HADOOP_MAPRED_PID_DIR:-$HADOOP_PID_DIR}" + HADOOP_MAPRED_PID_DIR="${HADOOP_PID_DIR}" + + HADOOP_ROOT_LOGGER="${HADOOP_MAPRED_ROOT_LOGGER:-INFO,console}" + HADOOP_MAPRED_ROOT_LOGGER="${HADOOP_ROOT_LOGGER}" + + HADOOP_MAPRED_HOME="${HADOOP_MAPRED_HOME:-$HADOOP_HOME_DIR}" + + HADOOP_IDENT_STRING="${HADOOP_MAPRED_IDENT_STRING:-$HADOOP_IDENT_STRING}" + HADOOP_MAPRED_IDENT_STRING="${HADOOP_IDENT_STRING}" +} + +if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then + _mc_this="${BASH_SOURCE-$0}" + HADOOP_LIBEXEC_DIR=$(cd -P -- "$(dirname -- "${_mc_this}")" >/dev/null && pwd -P) +fi -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -if [ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]; then +if [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" -elif [ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]; then - . "$HADOOP_COMMON_HOME"/libexec/hadoop-config.sh -elif [ -e "${HADOOP_COMMON_HOME}/bin/hadoop-config.sh" ]; then - . "$HADOOP_COMMON_HOME"/bin/hadoop-config.sh -elif [ -e "${HADOOP_HOME}/bin/hadoop-config.sh" ]; then - . "$HADOOP_HOME"/bin/hadoop-config.sh -elif [ -e "${HADOOP_MAPRED_HOME}/bin/hadoop-config.sh" ]; then - . "$HADOOP_MAPRED_HOME"/bin/hadoop-config.sh +elif [[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]]; then + . "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" +elif [[ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]]; then + . "${HADOOP_HOME}/libexec/hadoop-config.sh" else echo "Hadoop common not found." exit fi - -# Only set locally to use in HADOOP_OPTS. No need to export. -# The following defaults are useful when somebody directly invokes bin/mapred. -HADOOP_MAPRED_LOG_DIR=${HADOOP_MAPRED_LOG_DIR:-${HADOOP_MAPRED_HOME}/logs} -HADOOP_MAPRED_LOGFILE=${HADOOP_MAPRED_LOGFILE:-hadoop.log} -HADOOP_MAPRED_ROOT_LOGGER=${HADOOP_MAPRED_ROOT_LOGGER:-INFO,console} - -HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_MAPRED_LOG_DIR" -HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_MAPRED_LOGFILE" -export HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.root.logger=${HADOOP_MAPRED_ROOT_LOGGER}" - - Modified: hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh?rev=1619018&r1=1619017&r2=1619018&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh (original) +++ hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh Wed Aug 20 01:22:42 2014 @@ -15,133 +15,32 @@ # See the License for the specific language governing permissions and # limitations under the License. - -# -# Environment Variables -# -# HADOOP_JHS_LOGGER Hadoop JobSummary logger. -# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_MAPRED_HOME}/conf. -# HADOOP_MAPRED_PID_DIR The pid files are stored. /tmp by default. -# HADOOP_MAPRED_NICENESS The scheduling priority for daemons. Defaults to 0. -## - -usage="Usage: mr-jobhistory-daemon.sh [--config <conf-dir>] (start|stop) <mapred-command> " - -# if no args specified, show usage -if [ $# -le 1 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin"; pwd` - -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -if [ -e ${HADOOP_LIBEXEC_DIR}/mapred-config.sh ]; then - . $HADOOP_LIBEXEC_DIR/mapred-config.sh -fi - -# get arguments -startStop=$1 -shift -command=$1 -shift - -hadoop_rotate_log () +function hadoop_usage { - log=$1; - num=5; - if [ -n "$2" ]; then - num=$2 - fi - if [ -f "$log" ]; then # rotate logs - while [ $num -gt 1 ]; do - prev=`expr $num - 1` - [ -f "$log.$prev" ] && mv "$log.$prev" "$log.$num" - num=$prev - done - mv "$log" "$log.$num"; - fi + echo "Usage: mr-jobhistory-daemon.sh [--config confdir] (start|stop|status) <hadoop-command> <args...>" } -if [ "$HADOOP_MAPRED_IDENT_STRING" = "" ]; then - export HADOOP_MAPRED_IDENT_STRING="$USER" -fi - -export HADOOP_MAPRED_HOME=${HADOOP_MAPRED_HOME:-${HADOOP_PREFIX}} -export HADOOP_MAPRED_LOGFILE=mapred-$HADOOP_MAPRED_IDENT_STRING-$command-$HOSTNAME.log -export HADOOP_MAPRED_ROOT_LOGGER=${HADOOP_MAPRED_ROOT_LOGGER:-INFO,RFA} -export HADOOP_JHS_LOGGER=${HADOOP_JHS_LOGGER:-INFO,JSA} - -if [ -f "${HADOOP_CONF_DIR}/mapred-env.sh" ]; then - . "${HADOOP_CONF_DIR}/mapred-env.sh" -fi - -mkdir -p "$HADOOP_MAPRED_LOG_DIR" -chown $HADOOP_MAPRED_IDENT_STRING $HADOOP_MAPRED_LOG_DIR - -if [ "$HADOOP_MAPRED_PID_DIR" = "" ]; then - HADOOP_MAPRED_PID_DIR=/tmp -fi - -HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.id.str=$HADOOP_MAPRED_IDENT_STRING" - -log=$HADOOP_MAPRED_LOG_DIR/mapred-$HADOOP_MAPRED_IDENT_STRING-$command-$HOSTNAME.out -pid=$HADOOP_MAPRED_PID_DIR/mapred-$HADOOP_MAPRED_IDENT_STRING-$command.pid - -HADOOP_MAPRED_STOP_TIMEOUT=${HADOOP_MAPRED_STOP_TIMEOUT:-5} - -# Set default scheduling priority -if [ "$HADOOP_MAPRED_NICENESS" = "" ]; then - export HADOOP_MAPRED_NICENESS=0 +# let's locate libexec... +if [[ -n "${HADOOP_PREFIX}" ]]; then + DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +else + this="${BASH_SOURCE-$0}" + bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) + DEFAULT_LIBEXEC_DIR="${bin}/../libexec" +fi + +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/yarn-config.sh." 2>&1 + exit 1 fi -case $startStop in - - (start) - - mkdir -p "$HADOOP_MAPRED_PID_DIR" - - if [ -f $pid ]; then - if kill -0 `cat $pid` > /dev/null 2>&1; then - echo $command running as process `cat $pid`. Stop it first. - exit 1 - fi - fi - - hadoop_rotate_log $log - echo starting $command, logging to $log - cd "$HADOOP_MAPRED_HOME" - nohup nice -n $HADOOP_MAPRED_NICENESS "$HADOOP_MAPRED_HOME"/bin/mapred --config $HADOOP_CONF_DIR $command "$@" > "$log" 2>&1 < /dev/null & - echo $! > $pid - sleep 1; head "$log" - ;; - - (stop) - - if [ -f $pid ]; then - TARGET_PID=`cat $pid` - if kill -0 $TARGET_PID > /dev/null 2>&1; then - echo stopping $command - kill $TARGET_PID - sleep $HADOOP_MAPRED_STOP_TIMEOUT - if kill -0 $TARGET_PID > /dev/null 2>&1; then - echo "$command did not stop gracefully after $HADOOP_MAPRED_STOP_TIMEOUT seconds: killing with kill -9" - kill -9 $TARGET_PID - fi - else - echo no $command to stop - fi - rm -f $pid - else - echo no $command to stop - fi - ;; - - (*) - echo $usage - exit 1 - ;; +daemonmode=$1 +shift -esac +exec "${HADOOP_MAPRED_HOME}/bin/mapred" \ +--config "${HADOOP_CONF_DIR}" --daemon "${daemonmode}" "$@" Propchange: hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/conf/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1617528-1618997 Modified: hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/conf/mapred-env.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/conf/mapred-env.sh?rev=1619018&r1=1619017&r2=1619018&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/conf/mapred-env.sh (original) +++ hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/conf/mapred-env.sh Wed Aug 20 01:22:42 2014 @@ -13,15 +13,59 @@ # See the License for the specific language governing permissions and # limitations under the License. -# export JAVA_HOME=/home/y/libexec/jdk1.6.0/ +## +## THIS FILE ACTS AS AN OVERRIDE FOR hadoop-env.sh FOR ALL +## WORK DONE BY THE mapred AND RELATED COMMANDS. +## +## Precedence rules: +## +## mapred-env.sh > hadoop-env.sh > hard-coded defaults +## +## MAPRED_xyz > HADOOP_xyz > hard-coded defaults +## -export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=1000 +### +# Generic settings for MapReduce +### -export HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA +#Override the log4j settings for all MR apps +# export MAPRED_ROOT_LOGGER="INFO,console" +# Override Hadoop's log directory & file +# export HADOOP_MAPRED_LOG_DIR="" + +# Override Hadoop's pid directory +# export HADOOP_MAPRED_PID_DIR= + +# Override Hadoop's identity string. $USER by default. +# This is used in writing log and pid files, so keep that in mind! +# export HADOOP_MAPRED_IDENT_STRING=$USER + +# Override Hadoop's process priority +# Note that sub-processes will also run at this level! +# export HADOOP_MAPRED_NICENESS=0 + +### +# Job History Server specific parameters +### + +# Specify the max heapsize for the Job History Server using a numerical value +# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set +# the value to 1000. +# This value will be overridden by an Xmx setting specified in either +# MAPRED_OPTS, HADOOP_OPTS, and/or HADOOP_JOB_HISTORYSERVER_OPTS. +# If not specified, the default value will be picked from either YARN_HEAPMAX +# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. +# +#export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=1000 + +# Specify the JVM options to be used when starting the ResourceManager. +# These options will be appended to the options specified as YARN_OPTS +# and therefore may override any similar flags set in YARN_OPTS #export HADOOP_JOB_HISTORYSERVER_OPTS= -#export HADOOP_MAPRED_LOG_DIR="" # Where log files are stored. $HADOOP_MAPRED_HOME/logs by default. -#export HADOOP_JHS_LOGGER=INFO,RFA # Hadoop JobSummary logger. -#export HADOOP_MAPRED_PID_DIR= # The pid files are stored. /tmp by default. -#export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default -#export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0. + +# Specify the log4j settings for the JobHistoryServer +#export HADOOP_JHS_LOGGER=INFO,RFA + + + Modified: hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/OracleDBRecordReader.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/OracleDBRecordReader.java?rev=1619018&r1=1619017&r2=1619018&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/OracleDBRecordReader.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/OracleDBRecordReader.java Wed Aug 20 01:22:42 2014 @@ -81,15 +81,14 @@ public class OracleDBRecordReader<T exte try { DBInputFormat.DBInputSplit split = getSplit(); - if (split.getLength() > 0 && split.getStart() > 0){ + if (split.getLength() > 0){ String querystring = query.toString(); query = new StringBuilder(); query.append("SELECT * FROM (SELECT a.*,ROWNUM dbif_rno FROM ( "); query.append(querystring); - query.append(" ) a WHERE rownum <= ").append(split.getStart()); - query.append(" + ").append(split.getLength()); - query.append(" ) WHERE dbif_rno >= ").append(split.getStart()); + query.append(" ) a WHERE rownum <= ").append(split.getEnd()); + query.append(" ) WHERE dbif_rno > ").append(split.getStart()); } } catch (IOException ex) { // ignore, will not throw. Modified: hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestDbClasses.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestDbClasses.java?rev=1619018&r1=1619017&r2=1619018&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestDbClasses.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestDbClasses.java Wed Aug 20 01:22:42 2014 @@ -110,7 +110,7 @@ public class TestDbClasses { splitter, NullDBWritable.class, configuration, connect, dbConfiguration, "condition", fields, "table"); assertEquals( - "SELECT * FROM (SELECT a.*,ROWNUM dbif_rno FROM ( SELECT f1, f2 FROM table WHERE condition ORDER BY Order ) a WHERE rownum <= 1 + 9 ) WHERE dbif_rno >= 1", + "SELECT * FROM (SELECT a.*,ROWNUM dbif_rno FROM ( SELECT f1, f2 FROM table WHERE condition ORDER BY Order ) a WHERE rownum <= 10 ) WHERE dbif_rno > 1", recorder.getSelectQuery()); }