Author: wang Date: Wed Aug 20 01:22:42 2014 New Revision: 1619018 URL: http://svn.apache.org/r1619018 Log: Merge from trunk to branch.
Removed: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/slaves.sh Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/CHANGES.txt hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/yarn hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/CHANGES.txt?rev=1619018&r1=1619017&r2=1619018&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/CHANGES.txt Wed Aug 20 01:22:42 2014 @@ -211,6 +211,12 @@ Release 2.6.0 - UNRELEASED YARN-2397. Avoided loading two authentication filters for RM and TS web interfaces. (Varun Vasudev via zjshen) + YARN-2409. RM ActiveToStandBy transition missing stoping previous rmDispatcher. + (Rohith via jianhe) + + YARN-2249. Avoided AM release requests being lost on work preserving RM + restart. (Jian He via zjshen) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh?rev=1619018&r1=1619017&r2=1619018&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh Wed Aug 20 01:22:42 2014 @@ -16,20 +16,34 @@ # limitations under the License. -# Start all yarn daemons. Run this on master node. - -echo "starting yarn daemons" - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin"; pwd` - -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -. $HADOOP_LIBEXEC_DIR/yarn-config.sh +function hadoop_usage +{ + echo "Usage: start-yarn.sh [--config confdir]" +} + +this="${BASH_SOURCE-$0}" +bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) + +# let's locate libexec... +if [[ -n "${HADOOP_PREFIX}" ]]; then + DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +else + DEFAULT_LIBEXEC_DIR="${bin}/../libexec" +fi + +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/yarn-config.sh." 2>&1 + exit 1 +fi # start resourceManager -"$bin"/yarn-daemon.sh --config $YARN_CONF_DIR start resourcemanager +"${bin}/yarn-daemon.sh" --config "${YARN_CONF_DIR}" start resourcemanager # start nodeManager -"$bin"/yarn-daemons.sh --config $YARN_CONF_DIR start nodemanager +"${bin}/yarn-daemons.sh" --config "${YARN_CONF_DIR}" start nodemanager # start proxyserver -#"$bin"/yarn-daemon.sh --config $YARN_CONF_DIR start proxyserver +#"${bin}/yarn-daemon.sh" --config "${YARN_CONF_DIR}" start proxyserver Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh?rev=1619018&r1=1619017&r2=1619018&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh Wed Aug 20 01:22:42 2014 @@ -18,18 +18,34 @@ # Stop all yarn daemons. Run this on master node. -echo "stopping yarn daemons" +function hadoop_usage +{ + echo "Usage: stop-yarn.sh [--config confdir]" +} -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin"; pwd` +this="${BASH_SOURCE-$0}" +bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -. $HADOOP_LIBEXEC_DIR/yarn-config.sh - -# stop resourceManager -"$bin"/yarn-daemon.sh --config $YARN_CONF_DIR stop resourcemanager -# stop nodeManager -"$bin"/yarn-daemons.sh --config $YARN_CONF_DIR stop nodemanager -# stop proxy server -"$bin"/yarn-daemon.sh --config $YARN_CONF_DIR stop proxyserver +# let's locate libexec... +if [[ -n "${HADOOP_PREFIX}" ]]; then + DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +else + DEFAULT_LIBEXEC_DIR="${bin}/../libexec" +fi + +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/yarn-config.sh." 2>&1 + exit 1 +fi + +# start resourceManager +"${bin}/yarn-daemon.sh" --config "${YARN_CONF_DIR}" stop resourcemanager +# start nodeManager +"${bin}/yarn-daemons.sh" --config "${YARN_CONF_DIR}" stop nodemanager +# start proxyserver +#"${bin}/yarn-daemon.sh" --config "${YARN_CONF_DIR}" stop proxyserver Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/yarn URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/yarn?rev=1619018&r1=1619017&r2=1619018&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/yarn (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/yarn Wed Aug 20 01:22:42 2014 @@ -15,266 +15,182 @@ # See the License for the specific language governing permissions and # limitations under the License. - -# The Hadoop command script -# -# Environment Variables -# -# JAVA_HOME The java implementation to use. Overrides JAVA_HOME. -# -# YARN_USER_CLASSPATH Additional user CLASSPATH entries. -# -# YARN_USER_CLASSPATH_FIRST If set to non empty value then the user classpath -# specified in YARN_USER_CLASSPATH will be -# appended at the beginning of YARN's final -# classpath instead of at the end. -# -# YARN_HEAPSIZE The maximum amount of heap to use, in MB. -# Default is 1000. -# -# YARN_{COMMAND}_HEAPSIZE overrides YARN_HEAPSIZE for a given command -# eg YARN_NODEMANAGER_HEAPSIZE sets the heap -# size for the NodeManager. If you set the -# heap size in YARN_{COMMAND}_OPTS or YARN_OPTS -# they take precedence. -# -# YARN_OPTS Extra Java runtime options. -# -# YARN_CLIENT_OPTS when the respective command is run. -# YARN_{COMMAND}_OPTS etc YARN_NODEMANAGER_OPTS applies to NodeManager -# for e.g. YARN_CLIENT_OPTS applies to -# more than one command (fs, dfs, fsck, -# dfsadmin etc) -# -# YARN_CONF_DIR Alternate conf dir. Default is ${HADOOP_YARN_HOME}/conf. -# -# YARN_ROOT_LOGGER The root appender. Default is INFO,console -# - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin" > /dev/null; pwd` - -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -. $HADOOP_LIBEXEC_DIR/yarn-config.sh - -function print_usage(){ - echo "Usage: yarn [--config confdir] COMMAND" +function hadoop_usage +{ + echo "Usage: yarn [--config confdir] [--daemon (start|stop|status)] COMMAND" echo "where COMMAND is one of:" - echo " resourcemanager -format-state-store deletes the RMStateStore" - echo " resourcemanager run the ResourceManager" + echo " application prints application(s) report/kill application" + echo " applicationattempt prints applicationattempt(s) report" + echo " classpath prints the class path needed to get the" + echo " Hadoop jar and the required libraries" + echo " container prints container(s) report" + echo " daemonlog get/set the log level for each daemon" + echo " jar <jar> run a jar file" + echo " logs dump container logs" + echo " node prints node report(s)" echo " nodemanager run a nodemanager on each slave" - echo " timelineserver run the timeline server" + echo " proxyserver run the web app proxy server" + echo " resourcemanager run the ResourceManager" + echo " resourcemanager -format-state-store deletes the RMStateStore" echo " rmadmin admin tools" + echo " timelineserver run the timeline server" echo " version print the version" - echo " jar <jar> run a jar file" - echo " application prints application(s)" - echo " report/kill application" - echo " applicationattempt prints applicationattempt(s)" - echo " report" - echo " container prints container(s) report" - echo " node prints node report(s)" - echo " logs dump container logs" - echo " classpath prints the class path needed to" - echo " get the Hadoop jar and the" - echo " required libraries" - echo " daemonlog get/set the log level for each" - echo " daemon" echo " or" echo " CLASSNAME run the class named CLASSNAME" echo "Most commands print help when invoked w/o parameters." } -# if no args specified, show usage -if [ $# = 0 ]; then - print_usage + +# let's locate libexec... +if [[ -n "${HADOOP_PREFIX}" ]]; then + DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +else + this="${BASH_SOURCE-$0}" + bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) + DEFAULT_LIBEXEC_DIR="${bin}/../libexec" +fi + +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/yarn-config.sh." 2>&1 exit 1 fi +# if no args specified, show usage +if [[ $# = 0 ]]; then + hadoop_exit_with_usage 1 +fi + # get arguments COMMAND=$1 shift -case $COMMAND in - # usage flags - --help|-help|-h) - print_usage +case "${COMMAND}" in + application|applicationattempt|container) + CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI + YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}" + ;; + classpath) + hadoop_finalize + echo "${CLASSPATH}" exit - ;; + ;; + daemonlog) + CLASS=org.apache.hadoop.log.LogLevel + YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}" + ;; + jar) + CLASS=org.apache.hadoop.util.RunJar + YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}" + ;; + historyserver) + daemon="true" + echo "DEPRECATED: Use of this command to start the timeline server is deprecated." 1>&2 + echo "Instead use the timelineserver command for it." 1>&2 + echo "Starting the History Server anyway..." 1>&2 + CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer' + ;; + logs) + CLASS=org.apache.hadoop.yarn.logaggregation.LogDumper + YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}" + ;; + node) + CLASS=org.apache.hadoop.yarn.client.cli.NodeCLI + YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}" + ;; + nodemanager) + daemon="true" + CLASS='org.apache.hadoop.yarn.server.nodemanager.NodeManager' + YARN_OPTS="${YARN_OPTS} ${YARN_NODEMANAGER_OPTS}" + if [[ -n "${YARN_NODEMANAGER_HEAPSIZE}" ]]; then + JAVA_HEAP_MAX="-Xmx${YARN_NODEMANAGER_HEAPSIZE}m" + fi + ;; + proxyserver) + daemon="true" + CLASS='org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer' + YARN_OPTS="${YARN_OPTS} ${YARN_PROXYSERVER_OPTS}" + if [[ -n "${YARN_PROXYSERVER_HEAPSIZE}" ]]; then + JAVA_HEAP_MAX="-Xmx${YARN_PROXYSERVER_HEAPSIZE}m" + fi + ;; + resourcemanager) + daemon="true" + CLASS='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager' + YARN_OPTS="${YARN_OPTS} ${YARN_RESOURCEMANAGER_OPTS}" + if [[ -n "${YARN_RESOURCEMANAGER_HEAPSIZE}" ]]; then + JAVA_HEAP_MAX="-Xmx${YARN_RESOURCEMANAGER_HEAPSIZE}m" + fi + ;; + rmadmin) + CLASS='org.apache.hadoop.yarn.client.cli.RMAdminCLI' + YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}" + ;; + timelineserver) + daemon="true" + CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer' + YARN_OPTS="${YARN_OPTS} ${YARN_TIMELINESERVER_OPTS}" + if [[ -n "${YARN_TIMELINESERVER_HEAPSIZE}" ]]; then + JAVA_HEAP_MAX="-Xmx${YARN_TIMELINESERVER_HEAPSIZE}m" + fi + ;; + version) + CLASS=org.apache.hadoop.util.VersionInfo + YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}" + ;; + -*) + hadoop_exit_with_usage 1 + ;; + *) + CLASS="${COMMAND}" + ;; esac -if [ -f "${YARN_CONF_DIR}/yarn-env.sh" ]; then - . "${YARN_CONF_DIR}/yarn-env.sh" -fi +# set HADOOP_OPTS to YARN_OPTS so that we can use +# finalize, etc, without doing anything funky +HADOOP_OPTS="${YARN_OPTS}" -# some Java parameters -if [ "$JAVA_HOME" != "" ]; then - #echo "run java in $JAVA_HOME" - JAVA_HOME=$JAVA_HOME -fi - -if [ "$JAVA_HOME" = "" ]; then - echo "Error: JAVA_HOME is not set." - exit 1 -fi - -JAVA=$JAVA_HOME/bin/java -JAVA_HEAP_MAX=-Xmx1000m +daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out" +daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid" -# check envvars which might override default args -if [ "$YARN_HEAPSIZE" != "" ]; then - #echo "run with heapsize $YARN_HEAPSIZE" - JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m" - #echo $JAVA_HEAP_MAX +if [[ "${HADOOP_DAEMON_MODE}" != "default" ]]; then + # shellcheck disable=SC2034 + HADOOP_ROOT_LOGGER="${HADOOP_DAEMON_ROOT_LOGGER}" + YARN_ROOT_LOGGER="${HADOOP_DAEMON_ROOT_LOGGER}" + HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.log" fi -# CLASSPATH initially contains $HADOOP_CONF_DIR & $YARN_CONF_DIR -if [ ! -d "$HADOOP_CONF_DIR" ]; then - echo No HADOOP_CONF_DIR set. - echo Please specify it either in yarn-env.sh or in the environment. - exit 1 -fi +hadoop_add_param HADOOP_OPTS Xmx "${JAVA_HEAP_MAX}" -CLASSPATH="${HADOOP_CONF_DIR}:${YARN_CONF_DIR}:${CLASSPATH}" +# Add YARN custom options to comamnd line in case someone actaully +# used these. +# +# Note that we are replacing ' ' with '\ ' so that when we exec +# stuff it works +# +hadoop_add_param HADOOP_OPTS yarn.log.dir "-Dyarn.log.dir=${HADOOP_LOG_DIR/ /\ }" +hadoop_add_param HADOOP_OPTS yarn.log.file "-Dyarn.log.file=${HADOOP_LOGFILE/ /\ }" +hadoop_add_param HADOOP_OPTS yarn.home.dir "-Dyarn.home.dir=${HADOOP_YARN_HOME/ /\ }" +hadoop_add_param HADOOP_OPTS yarn.root.logger "-Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}" -# for developers, add Hadoop classes to CLASSPATH -if [ -d "$HADOOP_YARN_HOME/yarn-api/target/classes" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-api/target/classes -fi -if [ -d "$HADOOP_YARN_HOME/yarn-common/target/classes" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-common/target/classes -fi -if [ -d "$HADOOP_YARN_HOME/yarn-mapreduce/target/classes" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-mapreduce/target/classes -fi -if [ -d "$HADOOP_YARN_HOME/yarn-master-worker/target/classes" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-master-worker/target/classes -fi -if [ -d "$HADOOP_YARN_HOME/yarn-server/yarn-server-nodemanager/target/classes" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-server/yarn-server-nodemanager/target/classes -fi -if [ -d "$HADOOP_YARN_HOME/yarn-server/yarn-server-common/target/classes" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-server/yarn-server-common/target/classes -fi -if [ -d "$HADOOP_YARN_HOME/yarn-server/yarn-server-resourcemanager/target/classes" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-server/yarn-server-resourcemanager/target/classes -fi -if [ -d "$HADOOP_YARN_HOME/yarn-server/yarn-server-applicationhistoryservice/target/classes" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-server/yarn-server-applicationhistoryservice/target/classes -fi -if [ -d "$HADOOP_YARN_HOME/build/test/classes" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/target/test/classes -fi -if [ -d "$HADOOP_YARN_HOME/build/tools" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/build/tools -fi +hadoop_finalize -CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/${YARN_DIR}/* -CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/${YARN_LIB_JARS_DIR}/* +export CLASSPATH -# Add user defined YARN_USER_CLASSPATH to the class path (if defined) -if [ -n "$YARN_USER_CLASSPATH" ]; then - if [ -n "$YARN_USER_CLASSPATH_FIRST" ]; then - # User requested to add the custom entries at the beginning - CLASSPATH=${YARN_USER_CLASSPATH}:${CLASSPATH} +if [[ -n "${daemon}" ]]; then + if [[ -n "${secure_service}" ]]; then + hadoop_secure_daemon_handler "${HADOOP_DAEMON_MODE}" "${COMMAND}" \ + "${CLASS}" "${daemon_pidfile}" "${daemon_outfile}" \ + "${priv_pidfile}" "${priv_outfile}" "${priv_errfile}" "$@" else - # By default we will just append the extra entries at the end - CLASSPATH=${CLASSPATH}:${YARN_USER_CLASSPATH} + hadoop_daemon_handler "${HADOOP_DAEMON_MODE}" "${COMMAND}" "${CLASS}" \ + "${daemon_pidfile}" "${daemon_outfile}" "$@" fi -fi - -# so that filenames w/ spaces are handled correctly in loops below -IFS= - -# default log directory & file -if [ "$YARN_LOG_DIR" = "" ]; then - YARN_LOG_DIR="$HADOOP_YARN_HOME/logs" -fi -if [ "$YARN_LOGFILE" = "" ]; then - YARN_LOGFILE='yarn.log' -fi - -# restore ordinary behaviour -unset IFS - -# figure out which class to run -if [ "$COMMAND" = "classpath" ] ; then - echo $CLASSPATH - exit -elif [ "$COMMAND" = "rmadmin" ] ; then - CLASS='org.apache.hadoop.yarn.client.cli.RMAdminCLI' - YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS" -elif [ "$COMMAND" = "application" ] || - [ "$COMMAND" = "applicationattempt" ] || - [ "$COMMAND" = "container" ]; then - CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI - YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS" - set -- $COMMAND $@ -elif [ "$COMMAND" = "node" ] ; then - CLASS=org.apache.hadoop.yarn.client.cli.NodeCLI - YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS" -elif [ "$COMMAND" = "resourcemanager" ] ; then - CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/rm-config/log4j.properties - CLASS='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager' - YARN_OPTS="$YARN_OPTS $YARN_RESOURCEMANAGER_OPTS" - if [ "$YARN_RESOURCEMANAGER_HEAPSIZE" != "" ]; then - JAVA_HEAP_MAX="-Xmx""$YARN_RESOURCEMANAGER_HEAPSIZE""m" - fi -elif [ "$COMMAND" = "historyserver" ] ; then - echo "DEPRECATED: Use of this command to start the timeline server is deprecated." 1>&2 - echo "Instead use the timelineserver command for it." 1>&2 - CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/ahs-config/log4j.properties - CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer' - YARN_OPTS="$YARN_OPTS $YARN_HISTORYSERVER_OPTS" - if [ "$YARN_HISTORYSERVER_HEAPSIZE" != "" ]; then - JAVA_HEAP_MAX="-Xmx""$YARN_HISTORYSERVER_HEAPSIZE""m" - fi -elif [ "$COMMAND" = "timelineserver" ] ; then - CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/timelineserver-config/log4j.properties - CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer' - YARN_OPTS="$YARN_OPTS $YARN_TIMELINESERVER_OPTS" - if [ "$YARN_TIMELINESERVER_HEAPSIZE" != "" ]; then - JAVA_HEAP_MAX="-Xmx""$YARN_TIMELINESERVER_HEAPSIZE""m" - fi -elif [ "$COMMAND" = "nodemanager" ] ; then - CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/nm-config/log4j.properties - CLASS='org.apache.hadoop.yarn.server.nodemanager.NodeManager' - YARN_OPTS="$YARN_OPTS -server $YARN_NODEMANAGER_OPTS" - if [ "$YARN_NODEMANAGER_HEAPSIZE" != "" ]; then - JAVA_HEAP_MAX="-Xmx""$YARN_NODEMANAGER_HEAPSIZE""m" - fi -elif [ "$COMMAND" = "proxyserver" ] ; then - CLASS='org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer' - YARN_OPTS="$YARN_OPTS $YARN_PROXYSERVER_OPTS" - if [ "$YARN_PROXYSERVER_HEAPSIZE" != "" ]; then - JAVA_HEAP_MAX="-Xmx""$YARN_PROXYSERVER_HEAPSIZE""m" - fi -elif [ "$COMMAND" = "version" ] ; then - CLASS=org.apache.hadoop.util.VersionInfo - YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS" -elif [ "$COMMAND" = "jar" ] ; then - CLASS=org.apache.hadoop.util.RunJar - YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS" -elif [ "$COMMAND" = "logs" ] ; then - CLASS=org.apache.hadoop.yarn.client.cli.LogsCLI - YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS" -elif [ "$COMMAND" = "daemonlog" ] ; then - CLASS=org.apache.hadoop.log.LogLevel - YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS" + exit $? else - CLASS=$COMMAND + hadoop_java_exec "${COMMAND}" "${CLASS}" "$@" fi - -YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR" -YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR" -YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE" -YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE" -YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$HADOOP_YARN_HOME" -YARN_OPTS="$YARN_OPTS -Dhadoop.home.dir=$HADOOP_YARN_HOME" -YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}" -YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}" -if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then - YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" -fi - -exec "$JAVA" -Dproc_$COMMAND $JAVA_HEAP_MAX $YARN_OPTS -classpath "$CLASSPATH" $CLASS "$@" Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh?rev=1619018&r1=1619017&r2=1619018&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh Wed Aug 20 01:22:42 2014 @@ -13,53 +13,81 @@ # See the License for the specific language governing permissions and # limitations under the License. -# included in all the hadoop scripts with source command -# should not be executable directly -bin=`which "$0"` -bin=`dirname "${bin}"` -bin=`cd "$bin"; pwd` +function hadoop_subproject_init +{ + + # at some point in time, someone thought it would be a good idea to + # create separate vars for every subproject. *sigh* + # let's perform some overrides and setup some defaults for bw compat + # this way the common hadoop var's == subproject vars and can be + # used interchangeable from here on out + # ... + # this should get deprecated at some point. + + if [[ -e "${YARN_CONF_DIR}/yarn-env.sh" ]]; then + . "${YARN_CONF_DIR}/yarn-env.sh" + elif [[ -e "${HADOOP_CONF_DIR}/yarn-env.sh" ]]; then + . "${HADOOP_CONF_DIR}/yarn-env.sh" + fi + + if [[ -n "${YARN_CONF_DIR}" ]]; then + HADOOP_CONF_DIR="${YARN_CONF_DIR}" + fi + + YARN_CONF_DIR="${HADOOP_CONF_DIR}" + + # YARN_CONF_DIR needs precedence over HADOOP_CONF_DIR + # and the various jar dirs + hadoop_add_classpath "${YARN_CONF_DIR}" before + + HADOOP_LOG_DIR="${YARN_LOG_DIR:-$HADOOP_LOG_DIR}" + YARN_LOG_DIR="${HADOOP_LOG_DIR}" + + HADOOP_LOGFILE="${YARN_LOGFILE:-$HADOOP_LOGFILE}" + YARN_LOGFILE="${HADOOP_LOGFILE}" + + HADOOP_NICENESS="${YARN_NICENESS:-$HADOOP_NICENESS}" + YARN_NICENESS="${HADOOP_NICENESS}" + + HADOOP_STOP_TIMEOUT="${YARN_STOP_TIMEOUT:-$HADOOP_STOP_TIMEOUT}" + YARN_STOP_TIMEOUT="${HADOOP_STOP_TIMEOUT}" + + HADOOP_PID_DIR="${YARN_PID_DIR:-$HADOOP_PID_DIR}" + YARN_PID_DIR="${HADOOP_PID_DIR}" + + HADOOP_ROOT_LOGGER="${YARN_ROOT_LOGGER:-INFO,console}" + YARN_ROOT_LOGGER="${HADOOP_ROOT_LOGGER}" + + HADOOP_YARN_HOME="${HADOOP_YARN_HOME:-$HADOOP_PREFIX}" + + HADOOP_IDENT_STRING="${YARN_IDENT_STRING:-$HADOOP_IDENT_STRING}" + YARN_IDENT_STRING="${HADOOP_IDENT_STRING}" + + YARN_OPTS="${YARN_OPTS:-$HADOOP_OPTS}" + + # YARN-1429 added the completely superfluous YARN_USER_CLASSPATH + # env var. We're going to override HADOOP_USER_CLASSPATH to keep + # consistency with the rest of the duplicate/useless env vars + HADOOP_USER_CLASSPATH="${YARN_USER_CLASSPATH:-$HADOOP_USER_CLASSPATH}" + YARN_USER_CLASSPATH="${HADOOP_USER_CLASSPATH}" + + HADOOP_USER_CLASSPATH_FIRST="${YARN_USER_CLASSPATH_FIRST:-$HADOOP_USER_CLASSPATH_FIRST}" + YARN_USER_CLASSPATH_FIRST="${HADOOP_USER_CLASSPATH_FIRST}" +} -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -if [ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]; then - . ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh -elif [ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]; then - . "$HADOOP_COMMON_HOME"/libexec/hadoop-config.sh -elif [ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]; then - . "$HADOOP_HOME"/libexec/hadoop-config.sh +if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then + _yc_this="${BASH_SOURCE-$0}" + HADOOP_LIBEXEC_DIR=$(cd -P -- "$(dirname -- "${_yc_this}")" >/dev/null && pwd -P) +fi + +if [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" +elif [[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]]; then + . "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" +elif [[ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]]; then + . "${HADOOP_HOME}/libexec/hadoop-config.sh" else echo "Hadoop common not found." exit fi -# Same glibc bug that discovered in Hadoop. -# Without this you can see very large vmem settings on containers. -export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-4} - -#check to see if the conf dir is given as an optional argument -if [ $# -gt 1 ] -then - if [ "--config" = "$1" ] - then - shift - confdir=$1 - shift - YARN_CONF_DIR=$confdir - fi -fi - -# Allow alternate conf dir location. -export YARN_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_YARN_HOME/conf}" - -#check to see it is specified whether to use the slaves or the -# masters file -if [ $# -gt 1 ] -then - if [ "--hosts" = "$1" ] - then - shift - slavesfile=$1 - shift - export YARN_SLAVES="${YARN_CONF_DIR}/$slavesfile" - fi -fi Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh?rev=1619018&r1=1619017&r2=1619018&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh Wed Aug 20 01:22:42 2014 @@ -15,147 +15,32 @@ # See the License for the specific language governing permissions and # limitations under the License. - -# Runs a yarn command as a daemon. -# -# Environment Variables -# -# YARN_CONF_DIR Alternate conf dir. Default is ${HADOOP_YARN_HOME}/conf. -# YARN_LOG_DIR Where log files are stored. PWD by default. -# YARN_MASTER host:path where hadoop code should be rsync'd from -# YARN_PID_DIR The pid files are stored. /tmp by default. -# YARN_IDENT_STRING A string representing this instance of hadoop. $USER by default -# YARN_NICENESS The scheduling priority for daemons. Defaults to 0. -## - -usage="Usage: yarn-daemon.sh [--config <conf-dir>] [--hosts hostlistfile] (start|stop) <yarn-command> " - -# if no args specified, show usage -if [ $# -le 1 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin"; pwd` - -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -. $HADOOP_LIBEXEC_DIR/yarn-config.sh - -# get arguments -startStop=$1 -shift -command=$1 -shift - -hadoop_rotate_log () +function hadoop_usage { - log=$1; - num=5; - if [ -n "$2" ]; then - num=$2 - fi - if [ -f "$log" ]; then # rotate logs - while [ $num -gt 1 ]; do - prev=`expr $num - 1` - [ -f "$log.$prev" ] && mv "$log.$prev" "$log.$num" - num=$prev - done - mv "$log" "$log.$num"; - fi + echo "Usage: yarn-daemon.sh [--config confdir] (start|stop|status) <hadoop-command> <args...>" } -if [ -f "${YARN_CONF_DIR}/yarn-env.sh" ]; then - . "${YARN_CONF_DIR}/yarn-env.sh" -fi - -if [ "$YARN_IDENT_STRING" = "" ]; then - export YARN_IDENT_STRING="$USER" -fi - -# get log directory -if [ "$YARN_LOG_DIR" = "" ]; then - export YARN_LOG_DIR="$HADOOP_YARN_HOME/logs" -fi - -if [ ! -w "$YARN_LOG_DIR" ] ; then - mkdir -p "$YARN_LOG_DIR" - chown $YARN_IDENT_STRING $YARN_LOG_DIR -fi - -if [ "$YARN_PID_DIR" = "" ]; then - YARN_PID_DIR=/tmp -fi - -# some variables -export YARN_LOGFILE=yarn-$YARN_IDENT_STRING-$command-$HOSTNAME.log -export YARN_ROOT_LOGGER=${YARN_ROOT_LOGGER:-INFO,RFA} -log=$YARN_LOG_DIR/yarn-$YARN_IDENT_STRING-$command-$HOSTNAME.out -pid=$YARN_PID_DIR/yarn-$YARN_IDENT_STRING-$command.pid -YARN_STOP_TIMEOUT=${YARN_STOP_TIMEOUT:-5} - -# Set default scheduling priority -if [ "$YARN_NICENESS" = "" ]; then - export YARN_NICENESS=0 +# let's locate libexec... +if [[ -n "${HADOOP_PREFIX}" ]]; then + DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +else + this="${BASH_SOURCE-$0}" + bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) + DEFAULT_LIBEXEC_DIR="${bin}/../libexec" +fi + +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/yarn-config.sh." 2>&1 + exit 1 fi -case $startStop in - - (start) - - [ -w "$YARN_PID_DIR" ] || mkdir -p "$YARN_PID_DIR" - - if [ -f $pid ]; then - if kill -0 `cat $pid` > /dev/null 2>&1; then - echo $command running as process `cat $pid`. Stop it first. - exit 1 - fi - fi - - if [ "$YARN_MASTER" != "" ]; then - echo rsync from $YARN_MASTER - rsync -a -e ssh --delete --exclude=.svn --exclude='logs/*' --exclude='contrib/hod/logs/*' $YARN_MASTER/ "$HADOOP_YARN_HOME" - fi - - hadoop_rotate_log $log - echo starting $command, logging to $log - cd "$HADOOP_YARN_HOME" - nohup nice -n $YARN_NICENESS "$HADOOP_YARN_HOME"/bin/yarn --config $YARN_CONF_DIR $command "$@" > "$log" 2>&1 < /dev/null & - echo $! > $pid - sleep 1 - head "$log" - # capture the ulimit output - echo "ulimit -a" >> $log - ulimit -a >> $log 2>&1 - ;; - - (stop) - - if [ -f $pid ]; then - TARGET_PID=`cat $pid` - if kill -0 $TARGET_PID > /dev/null 2>&1; then - echo stopping $command - kill $TARGET_PID - sleep $YARN_STOP_TIMEOUT - if kill -0 $TARGET_PID > /dev/null 2>&1; then - echo "$command did not stop gracefully after $YARN_STOP_TIMEOUT seconds: killing with kill -9" - kill -9 $TARGET_PID - fi - else - echo no $command to stop - fi - rm -f $pid - else - echo no $command to stop - fi - ;; - - (*) - echo $usage - exit 1 - ;; - -esac - +daemonmode=$1 +shift +exec "${HADOOP_YARN_HOME}/bin/yarn" \ +--config "${HADOOP_CONF_DIR}" --daemon "${daemonmode}" "$@" Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh?rev=1619018&r1=1619017&r2=1619018&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh Wed Aug 20 01:22:42 2014 @@ -16,23 +16,31 @@ # limitations under the License. -# Run a Yarn command on all slave hosts. - -usage="Usage: yarn-daemons.sh [--config confdir] [--hosts hostlistfile] [start -|stop] command args..." +function hadoop_usage +{ + echo "Usage: yarn-daemons.sh [--config confdir] [--hosts hostlistfile] (start|stop|status) <yarn-command> <args...>" +} + +this="${BASH_SOURCE-$0}" +bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) + +# let's locate libexec... +if [[ -n "${HADOOP_PREFIX}" ]]; then + DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +else + DEFAULT_LIBEXEC_DIR="${bin}/../libexec" +fi -# if no args specified, show usage -if [ $# -le 1 ]; then - echo $usage +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/yarn-config.sh." 2>&1 exit 1 fi -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin"; pwd` - -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -. $HADOOP_LIBEXEC_DIR/yarn-config.sh - -exec "$bin/slaves.sh" --config $YARN_CONF_DIR cd "$HADOOP_YARN_HOME" \; "$bin/yarn-daemon.sh" --config $YARN_CONF_DIR "$@" +hadoop_connect_to_hosts "${bin}/yarn-daemon.sh" \ +--config "${HADOOP_CONF_DIR}" "$@" Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh?rev=1619018&r1=1619017&r2=1619018&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh Wed Aug 20 01:22:42 2014 @@ -13,118 +13,115 @@ # See the License for the specific language governing permissions and # limitations under the License. +## +## THIS FILE ACTS AS AN OVERRIDE FOR hadoop-env.sh FOR ALL +## WORK DONE BY THE yarn AND RELATED COMMANDS. +## +## Precedence rules: +## +## yarn-env.sh > hadoop-env.sh > hard-coded defaults +## +## YARN_xyz > HADOOP_xyz > hard-coded defaults +## + +### +# Generic settings for YARN +### + # User for YARN daemons export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn} -# resolve links - $0 may be a softlink -export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}" +# +# By default, YARN will use HADOOP_CONF_DIR. Specify a custom +# YARN_CONF_DIR here +# export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}" +# + +# Override Hadoop's log directory & file +# export YARN_LOG_DIR="$HADOOP_YARN_HOME/logs" +# export YARN_LOGFILE='yarn.log' + +# Need a custom-to-YARN service-level authorization policy file? +# export YARN_POLICYFILE="yarn-policy.xml" -# some Java parameters -# export JAVA_HOME=/home/y/libexec/jdk1.6.0/ -if [ "$JAVA_HOME" != "" ]; then - #echo "run java in $JAVA_HOME" - JAVA_HOME=$JAVA_HOME -fi - -if [ "$JAVA_HOME" = "" ]; then - echo "Error: JAVA_HOME is not set." - exit 1 -fi - -JAVA=$JAVA_HOME/bin/java -JAVA_HEAP_MAX=-Xmx1000m - -# For setting YARN specific HEAP sizes please use this -# Parameter and set appropriately -# YARN_HEAPSIZE=1000 - -# check envvars which might override default args -if [ "$YARN_HEAPSIZE" != "" ]; then - JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m" -fi +#Override the log4j settings for all YARN apps +# export YARN_ROOT_LOGGER="INFO,console" +### # Resource Manager specific parameters +### -# Specify the max Heapsize for the ResourceManager using a numerical value +# Specify the max heapsize for the ResourceManager using a numerical value # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set # the value to 1000. -# This value will be overridden by an Xmx setting specified in either YARN_OPTS -# and/or YARN_RESOURCEMANAGER_OPTS. +# This value will be overridden by an Xmx setting specified in either YARN_OPTS, +# HADOOP_OPTS, and/or YARN_RESOURCEMANAGER_OPTS. # If not specified, the default value will be picked from either YARN_HEAPMAX # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. +# #export YARN_RESOURCEMANAGER_HEAPSIZE=1000 -# Specify the max Heapsize for the timeline server using a numerical value -# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set -# the value to 1000. -# This value will be overridden by an Xmx setting specified in either YARN_OPTS -# and/or YARN_TIMELINESERVER_OPTS. -# If not specified, the default value will be picked from either YARN_HEAPMAX -# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. -#export YARN_TIMELINESERVER_HEAPSIZE=1000 - # Specify the JVM options to be used when starting the ResourceManager. # These options will be appended to the options specified as YARN_OPTS # and therefore may override any similar flags set in YARN_OPTS -#export YARN_RESOURCEMANAGER_OPTS= +# +# Examples for a Sun/Oracle JDK: +# a) override the appsummary log file: +# export YARN_RESOURCEMANAGER_OPTS="-Dyarn.server.resourcemanager.appsummary.log.file=rm-appsummary.log -Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY" +# +# b) Set JMX options +# export YARN_RESOURCEMANAGER_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=1026" +# +# c) Set garbage collection logs from hadoop-env.sh +# export YARN_RESOURCE_MANAGER_OPTS="${HADOOP_GC_SETTINGS} -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')" +# +# d) ... or set them directly +# export YARN_RESOURCEMANAGER_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')" +# +# +# export YARN_RESOURCEMANAGER_OPTS= +### # Node Manager specific parameters +### # Specify the max Heapsize for the NodeManager using a numerical value # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set # the value to 1000. -# This value will be overridden by an Xmx setting specified in either YARN_OPTS -# and/or YARN_NODEMANAGER_OPTS. +# This value will be overridden by an Xmx setting specified in either YARN_OPTS, +# HADOOP_OPTS, and/or YARN_NODEMANAGER_OPTS. # If not specified, the default value will be picked from either YARN_HEAPMAX # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. +# #export YARN_NODEMANAGER_HEAPSIZE=1000 # Specify the JVM options to be used when starting the NodeManager. # These options will be appended to the options specified as YARN_OPTS # and therefore may override any similar flags set in YARN_OPTS +# +# See ResourceManager for some examples +# #export YARN_NODEMANAGER_OPTS= -# so that filenames w/ spaces are handled correctly in loops below -IFS= +### +# TimeLineServer specifc parameters +### +# Specify the max Heapsize for the timeline server using a numerical value +# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set +# the value to 1000. +# This value will be overridden by an Xmx setting specified in either YARN_OPTS, +# HADOOP_OPTS, and/or YARN_TIMELINESERVER_OPTS. +# If not specified, the default value will be picked from either YARN_HEAPMAX +# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. +# +#export YARN_TIMELINESERVER_HEAPSIZE=1000 -# default log directory & file -if [ "$YARN_LOG_DIR" = "" ]; then - YARN_LOG_DIR="$HADOOP_YARN_HOME/logs" -fi -if [ "$YARN_LOGFILE" = "" ]; then - YARN_LOGFILE='yarn.log' -fi - -# default policy file for service-level authorization -if [ "$YARN_POLICYFILE" = "" ]; then - YARN_POLICYFILE="hadoop-policy.xml" -fi - -# restore ordinary behaviour -unset IFS - -MAC_OSX=false -case "`uname`" in -Darwin*) MAC_OSX=true;; -esac - -if $MAC_OSX; then - YARN_OPTS="$YARN_OPTS -Djava.security.krb5.realm= -Djava.security.krb5.kdc=" -fi - - -YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR" -YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR" -YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE" -YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE" -YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME" -YARN_OPTS="$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING" -YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}" -YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}" -if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then - YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" -fi -YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE" - +# Specify the JVM options to be used when starting the TimeLineServer. +# These options will be appended to the options specified as YARN_OPTS +# and therefore may override any similar flags set in YARN_OPTS +# +# See ResourceManager for some examples +# +#export YARN_TIMELINESERVER_OPTS= Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1619018&r1=1619017&r2=1619018&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Wed Aug 20 01:22:42 2014 @@ -1161,6 +1161,9 @@ public class ResourceManager extends Com ((Service)dispatcher).init(this.conf); ((Service)dispatcher).start(); removeService((Service)rmDispatcher); + // Need to stop previous rmDispatcher before assigning new dispatcher + // otherwise causes "AsyncDispatcher event handler" thread leak + ((Service) rmDispatcher).stop(); rmDispatcher = dispatcher; addIfService(rmDispatcher); rmContext.setDispatcher(rmDispatcher); Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java?rev=1619018&r1=1619017&r2=1619018&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java Wed Aug 20 01:22:42 2014 @@ -23,10 +23,14 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.Timer; +import java.util.TimerTask; import java.util.concurrent.ConcurrentHashMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -34,18 +38,25 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; +import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; +import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMoveEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerFinishedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerRecoverEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; @@ -54,6 +65,7 @@ import org.apache.hadoop.yarn.util.resou import com.google.common.util.concurrent.SettableFuture; + @SuppressWarnings("unchecked") public abstract class AbstractYarnScheduler <T extends SchedulerApplicationAttempt, N extends SchedulerNode> @@ -72,6 +84,7 @@ public abstract class AbstractYarnSchedu protected RMContext rmContext; protected Map<ApplicationId, SchedulerApplication<T>> applications; + protected int nmExpireInterval; protected final static List<Container> EMPTY_CONTAINER_LIST = new ArrayList<Container>(); @@ -87,6 +100,15 @@ public abstract class AbstractYarnSchedu super(name); } + @Override + public void serviceInit(Configuration conf) throws Exception { + nmExpireInterval = + conf.getInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, + YarnConfiguration.DEFAULT_RM_NM_EXPIRY_INTERVAL_MS); + createReleaseCache(); + super.serviceInit(conf); + } + public synchronized List<Container> getTransferredContainers( ApplicationAttemptId currentAttempt) { ApplicationId appId = currentAttempt.getApplicationId(); @@ -281,6 +303,19 @@ public abstract class AbstractYarnSchedu ((RMContainerImpl)rmContainer).setAMContainer(true); } } + + synchronized (schedulerAttempt) { + Set<ContainerId> releases = schedulerAttempt.getPendingRelease(); + if (releases.contains(container.getContainerId())) { + // release the container + rmContainer.handle(new RMContainerFinishedEvent(container + .getContainerId(), SchedulerUtils.createAbnormalContainerStatus( + container.getContainerId(), SchedulerUtils.RELEASED_CONTAINER), + RMContainerEventType.RELEASED)); + releases.remove(container.getContainerId()); + LOG.info(container.getContainerId() + " is released by application."); + } + } } } @@ -320,6 +355,62 @@ public abstract class AbstractYarnSchedu } } + protected void createReleaseCache() { + // Cleanup the cache after nm expire interval. + new Timer().schedule(new TimerTask() { + @Override + public void run() { + for (SchedulerApplication<T> app : applications.values()) { + + T attempt = app.getCurrentAppAttempt(); + synchronized (attempt) { + for (ContainerId containerId : attempt.getPendingRelease()) { + RMAuditLogger.logFailure( + app.getUser(), + AuditConstants.RELEASE_CONTAINER, + "Unauthorized access or invalid container", + "Scheduler", + "Trying to release container not owned by app or with invalid id.", + attempt.getApplicationId(), containerId); + } + attempt.getPendingRelease().clear(); + } + } + LOG.info("Release request cache is cleaned up"); + } + }, nmExpireInterval); + } + + // clean up a completed container + protected abstract void completedContainer(RMContainer rmContainer, + ContainerStatus containerStatus, RMContainerEventType event); + + protected void releaseContainers(List<ContainerId> containers, + SchedulerApplicationAttempt attempt) { + for (ContainerId containerId : containers) { + RMContainer rmContainer = getRMContainer(containerId); + if (rmContainer == null) { + if (System.currentTimeMillis() - ResourceManager.getClusterTimeStamp() + < nmExpireInterval) { + LOG.info(containerId + " doesn't exist. Add the container" + + " to the release request cache as it maybe on recovery."); + synchronized (attempt) { + attempt.getPendingRelease().add(containerId); + } + } else { + RMAuditLogger.logFailure(attempt.getUser(), + AuditConstants.RELEASE_CONTAINER, + "Unauthorized access or invalid container", "Scheduler", + "Trying to release container not owned by app or with invalid id.", + attempt.getApplicationId(), containerId); + } + } + completedContainer(rmContainer, + SchedulerUtils.createAbnormalContainerStatus(containerId, + SchedulerUtils.RELEASED_CONTAINER), RMContainerEventType.RELEASED); + } + } + public SchedulerNode getSchedulerNode(NodeId nodeId) { return nodes.get(nodeId); } Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java?rev=1619018&r1=1619017&r2=1619018&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java Wed Aug 20 01:22:42 2014 @@ -17,13 +17,14 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; -import com.google.common.base.Preconditions; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -41,7 +42,6 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; @@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent; import org.apache.hadoop.yarn.util.resource.Resources; +import com.google.common.base.Preconditions; import com.google.common.collect.HashMultiset; import com.google.common.collect.Multiset; @@ -87,6 +88,13 @@ public class SchedulerApplicationAttempt protected List<RMContainer> newlyAllocatedContainers = new ArrayList<RMContainer>(); + // This pendingRelease is used in work-preserving recovery scenario to keep + // track of the AM's outstanding release requests. RM on recovery could + // receive the release request form AM before it receives the container status + // from NM for recovery. In this case, the to-be-recovered containers reported + // by NM should not be recovered. + private Set<ContainerId> pendingRelease = null; + /** * Count how many times the application has been given an opportunity * to schedule a task at each priority. Each time the scheduler @@ -114,7 +122,7 @@ public class SchedulerApplicationAttempt new AppSchedulingInfo(applicationAttemptId, user, queue, activeUsersManager, rmContext.getEpoch()); this.queue = queue; - + this.pendingRelease = new HashSet<ContainerId>(); if (rmContext.getRMApps() != null && rmContext.getRMApps() .containsKey(applicationAttemptId.getApplicationId())) { @@ -163,6 +171,10 @@ public class SchedulerApplicationAttempt return appSchedulingInfo.getResourceRequests(priority); } + public Set<ContainerId> getPendingRelease() { + return this.pendingRelease; + } + public int getNewContainerId() { return appSchedulingInfo.getNewContainerId(); } Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1619018&r1=1619017&r2=1619018&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Wed Aug 20 01:22:42 2014 @@ -54,8 +54,6 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; -import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.*; @@ -199,7 +197,7 @@ public class CapacityScheduler extends private static final long DEFAULT_ASYNC_SCHEDULER_INTERVAL = 5; private boolean overrideWithQueueMappings = false; - private List<QueueMapping> mappings = new ArrayList<QueueMapping>(); + private List<QueueMapping> mappings = null; private Groups groups; @VisibleForTesting @@ -789,21 +787,7 @@ public class CapacityScheduler extends getMinimumResourceCapability(), maximumAllocation); // Release containers - for (ContainerId releasedContainerId : release) { - RMContainer rmContainer = getRMContainer(releasedContainerId); - if (rmContainer == null) { - RMAuditLogger.logFailure(application.getUser(), - AuditConstants.RELEASE_CONTAINER, - "Unauthorized access or invalid container", "CapacityScheduler", - "Trying to release container not owned by app or with invalid id", - application.getApplicationId(), releasedContainerId); - } - completedContainer(rmContainer, - SchedulerUtils.createAbnormalContainerStatus( - releasedContainerId, - SchedulerUtils.RELEASED_CONTAINER), - RMContainerEventType.RELEASED); - } + releaseContainers(release, application); synchronized (application) { @@ -1098,7 +1082,8 @@ public class CapacityScheduler extends } @Lock(CapacityScheduler.class) - private synchronized void completedContainer(RMContainer rmContainer, + @Override + protected synchronized void completedContainer(RMContainer rmContainer, ContainerStatus containerStatus, RMContainerEventType event) { if (rmContainer == null) { LOG.info("Null container completed..."); Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1619018&r1=1619017&r2=1619018&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java Wed Aug 20 01:22:42 2014 @@ -49,8 +49,6 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; -import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; @@ -810,7 +808,8 @@ public class FairScheduler extends /** * Clean up a completed container. */ - private synchronized void completedContainer(RMContainer rmContainer, + @Override + protected synchronized void completedContainer(RMContainer rmContainer, ContainerStatus containerStatus, RMContainerEventType event) { if (rmContainer == null) { LOG.info("Null container completed..."); @@ -913,21 +912,7 @@ public class FairScheduler extends } // Release containers - for (ContainerId releasedContainerId : release) { - RMContainer rmContainer = getRMContainer(releasedContainerId); - if (rmContainer == null) { - RMAuditLogger.logFailure(application.getUser(), - AuditConstants.RELEASE_CONTAINER, - "Unauthorized access or invalid container", "FairScheduler", - "Trying to release container not owned by app or with invalid id", - application.getApplicationId(), releasedContainerId); - } - completedContainer(rmContainer, - SchedulerUtils.createAbnormalContainerStatus( - releasedContainerId, - SchedulerUtils.RELEASED_CONTAINER), - RMContainerEventType.RELEASED); - } + releaseContainers(release, application); synchronized (application) { if (!ask.isEmpty()) { Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1619018&r1=1619017&r2=1619018&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Wed Aug 20 01:22:42 2014 @@ -52,8 +52,6 @@ import org.apache.hadoop.yarn.conf.YarnC import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; -import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; -import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; @@ -89,7 +87,6 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; - import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.server.utils.Lock; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; @@ -295,21 +292,7 @@ public class FifoScheduler extends clusterResource, minimumAllocation, maximumAllocation); // Release containers - for (ContainerId releasedContainer : release) { - RMContainer rmContainer = getRMContainer(releasedContainer); - if (rmContainer == null) { - RMAuditLogger.logFailure(application.getUser(), - AuditConstants.RELEASE_CONTAINER, - "Unauthorized access or invalid container", "FifoScheduler", - "Trying to release container not owned by app or with invalid id", - application.getApplicationId(), releasedContainer); - } - containerCompleted(rmContainer, - SchedulerUtils.createAbnormalContainerStatus( - releasedContainer, - SchedulerUtils.RELEASED_CONTAINER), - RMContainerEventType.RELEASED); - } + releaseContainers(release, application); synchronized (application) { @@ -443,7 +426,7 @@ public class FifoScheduler extends LOG.info("Skip killing " + container.getContainerId()); continue; } - containerCompleted(container, + completedContainer(container, SchedulerUtils.createAbnormalContainerStatus( container.getContainerId(), SchedulerUtils.COMPLETED_APPLICATION), RMContainerEventType.KILL); @@ -717,7 +700,7 @@ public class FifoScheduler extends for (ContainerStatus completedContainer : completedContainers) { ContainerId containerId = completedContainer.getContainerId(); LOG.debug("Container FINISHED: " + containerId); - containerCompleted(getRMContainer(containerId), + completedContainer(getRMContainer(containerId), completedContainer, RMContainerEventType.FINISHED); } @@ -818,7 +801,7 @@ public class FifoScheduler extends ContainerExpiredSchedulerEvent containerExpiredEvent = (ContainerExpiredSchedulerEvent) event; ContainerId containerid = containerExpiredEvent.getContainerId(); - containerCompleted(getRMContainer(containerid), + completedContainer(getRMContainer(containerid), SchedulerUtils.createAbnormalContainerStatus( containerid, SchedulerUtils.EXPIRED_CONTAINER), @@ -831,7 +814,8 @@ public class FifoScheduler extends } @Lock(FifoScheduler.class) - private synchronized void containerCompleted(RMContainer rmContainer, + @Override + protected synchronized void completedContainer(RMContainer rmContainer, ContainerStatus containerStatus, RMContainerEventType event) { if (rmContainer == null) { LOG.info("Null container completed..."); @@ -881,7 +865,7 @@ public class FifoScheduler extends } // Kill running containers for(RMContainer container : node.getRunningContainers()) { - containerCompleted(container, + completedContainer(container, SchedulerUtils.createAbnormalContainerStatus( container.getContainerId(), SchedulerUtils.LOST_CONTAINER), Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java?rev=1619018&r1=1619017&r2=1619018&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java Wed Aug 20 01:22:42 2014 @@ -49,7 +49,7 @@ public class MockAM { private volatile int responseId = 0; private final ApplicationAttemptId attemptId; - private final RMContext context; + private RMContext context; private ApplicationMasterProtocol amRMProtocol; private final List<ResourceRequest> requests = new ArrayList<ResourceRequest>(); @@ -61,8 +61,10 @@ public class MockAM { this.amRMProtocol = amRMProtocol; this.attemptId = attemptId; } - - void setAMRMProtocol(ApplicationMasterProtocol amRMProtocol) { + + public void setAMRMProtocol(ApplicationMasterProtocol amRMProtocol, + RMContext context) { + this.context = context; this.amRMProtocol = amRMProtocol; } Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java?rev=1619018&r1=1619017&r2=1619018&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java Wed Aug 20 01:22:42 2014 @@ -171,7 +171,6 @@ public class TestApplicationMasterServic RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); am1.registerAppAttempt(); - am1.setAMRMProtocol(rm.getApplicationMasterService()); AllocateRequestPBImpl allocateRequest = new AllocateRequestPBImpl(); List<ContainerId> release = new ArrayList<ContainerId>(); Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java?rev=1619018&r1=1619017&r2=1619018&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java Wed Aug 20 01:22:42 2014 @@ -331,6 +331,10 @@ public class TestRMHA { rm.adminService.transitionToStandby(requestInfo); rm.adminService.transitionToActive(requestInfo); rm.adminService.transitionToStandby(requestInfo); + + MyCountingDispatcher dispatcher = + (MyCountingDispatcher) rm.getRMContext().getDispatcher(); + assertTrue(!dispatcher.isStopped()); rm.adminService.transitionToActive(requestInfo); assertEquals(errorMessageForEventHandler, expectedEventHandlerCount, @@ -339,6 +343,11 @@ public class TestRMHA { assertEquals(errorMessageForService, expectedServiceCount, rm.getServices().size()); + + // Keep the dispatcher reference before transitioning to standby + dispatcher = (MyCountingDispatcher) rm.getRMContext().getDispatcher(); + + rm.adminService.transitionToStandby(requestInfo); assertEquals(errorMessageForEventHandler, expectedEventHandlerCount, ((MyCountingDispatcher) rm.getRMContext().getDispatcher()) @@ -346,6 +355,8 @@ public class TestRMHA { assertEquals(errorMessageForService, expectedServiceCount, rm.getServices().size()); + assertTrue(dispatcher.isStopped()); + rm.stop(); } @@ -492,6 +503,8 @@ public class TestRMHA { private int eventHandlerCount; + private volatile boolean stopped = false; + public MyCountingDispatcher() { super("MyCountingDispatcher"); this.eventHandlerCount = 0; @@ -510,5 +523,15 @@ public class TestRMHA { public int getEventHandlerCount() { return this.eventHandlerCount; } + + @Override + protected void serviceStop() throws Exception { + this.stopped = true; + super.serviceStop(); + } + + public boolean isStopped() { + return this.stopped; + } } } Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java?rev=1619018&r1=1619017&r2=1619018&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java Wed Aug 20 01:22:42 2014 @@ -289,7 +289,7 @@ public class TestRMRestart { // verify old AM is not accepted // change running AM to talk to new RM - am1.setAMRMProtocol(rm2.getApplicationMasterService()); + am1.setAMRMProtocol(rm2.getApplicationMasterService(), rm2.getRMContext()); AllocateResponse allocResponse = am1.allocate( new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()); @@ -1663,7 +1663,7 @@ public class TestRMRestart { nm1.setResourceTrackerService(rm2.getResourceTrackerService()); // recover app RMApp loadedApp1 = rm2.getRMContext().getRMApps().get(app1.getApplicationId()); - am1.setAMRMProtocol(rm2.getApplicationMasterService()); + am1.setAMRMProtocol(rm2.getApplicationMasterService(), rm2.getRMContext()); am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()); nm1.nodeHeartbeat(true); nm1 = new MockNM("127.0.0.1:1234", 15120, rm2.getResourceTrackerService()); Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java?rev=1619018&r1=1619017&r2=1619018&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java Wed Aug 20 01:22:42 2014 @@ -33,10 +33,13 @@ import java.util.Set; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; @@ -72,6 +75,9 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import com.google.common.base.Supplier; + + @SuppressWarnings({"rawtypes", "unchecked"}) @RunWith(value = Parameterized.class) public class TestWorkPreservingRMRestart { @@ -572,8 +578,8 @@ public class TestWorkPreservingRMRestart rm2.waitForState(app0.getApplicationId(), RMAppState.ACCEPTED); rm2.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.LAUNCHED); - am0.setAMRMProtocol(rm2.getApplicationMasterService()); - am0.registerAppAttempt(false); + am0.setAMRMProtocol(rm2.getApplicationMasterService(), rm2.getRMContext()); + am0.registerAppAttempt(true); rm2.waitForState(app0.getApplicationId(), RMAppState.RUNNING); rm2.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.RUNNING); @@ -646,6 +652,69 @@ public class TestWorkPreservingRMRestart waitForNumContainersToRecover(2, rm2, am0.getApplicationAttemptId()); } + // Test if RM on recovery receives the container release request from AM + // before it receives the container status reported by NM for recovery. this + // container should not be recovered. + @Test (timeout = 30000) + public void testReleasedContainerNotRecovered() throws Exception { + MemoryRMStateStore memStore = new MemoryRMStateStore(); + memStore.init(conf); + rm1 = new MockRM(conf, memStore); + MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService()); + nm1.registerNode(); + rm1.start(); + + RMApp app1 = rm1.submitApp(1024); + final MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); + + // Re-start RM + conf.setInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 8000); + rm2 = new MockRM(conf, memStore); + rm2.start(); + nm1.setResourceTrackerService(rm2.getResourceTrackerService()); + rm2.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED); + am1.setAMRMProtocol(rm2.getApplicationMasterService(), rm2.getRMContext()); + am1.registerAppAttempt(true); + + // try to release a container before the container is actually recovered. + final ContainerId runningContainer = + ContainerId.newInstance(am1.getApplicationAttemptId(), 2); + am1.allocate(null, Arrays.asList(runningContainer)); + + // send container statuses to recover the containers + List<NMContainerStatus> containerStatuses = + createNMContainerStatusForApp(am1); + nm1.registerNode(containerStatuses, null); + + // only the am container should be recovered. + waitForNumContainersToRecover(1, rm2, am1.getApplicationAttemptId()); + + final AbstractYarnScheduler scheduler = + (AbstractYarnScheduler) rm2.getResourceScheduler(); + // cached release request is cleaned. + // assertFalse(scheduler.getPendingRelease().contains(runningContainer)); + + AllocateResponse response = am1.allocate(null, null); + // AM gets notified of the completed container. + boolean receivedCompletedContainer = false; + for (ContainerStatus status : response.getCompletedContainersStatuses()) { + if (status.getContainerId().equals(runningContainer)) { + receivedCompletedContainer = true; + } + } + assertTrue(receivedCompletedContainer); + + GenericTestUtils.waitFor(new Supplier<Boolean>() { + public Boolean get() { + // release cache is cleaned up and previous running container is not + // recovered + return scheduler.getApplicationAttempt(am1.getApplicationAttemptId()) + .getPendingRelease().isEmpty() + && scheduler.getRMContainer(runningContainer) == null; + } + }, 1000, 20000); + } + private void asserteMetrics(QueueMetrics qm, int appsSubmitted, int appsPending, int appsRunning, int appsCompleted, int allocatedContainers, int availableMB, int availableVirtualCores, @@ -661,7 +730,7 @@ public class TestWorkPreservingRMRestart assertEquals(allocatedVirtualCores, qm.getAllocatedVirtualCores()); } - private void waitForNumContainersToRecover(int num, MockRM rm, + public static void waitForNumContainersToRecover(int num, MockRM rm, ApplicationAttemptId attemptId) throws Exception { AbstractYarnScheduler scheduler = (AbstractYarnScheduler) rm.getResourceScheduler(); @@ -674,7 +743,9 @@ public class TestWorkPreservingRMRestart attempt = scheduler.getApplicationAttempt(attemptId); } while (attempt.getLiveContainers().size() < num) { - System.out.println("Wait for " + num + " containers to recover."); + System.out.println("Wait for " + num + + " containers to recover. currently: " + + attempt.getLiveContainers().size()); Thread.sleep(200); } }