svn commit: r1425195 - in /hadoop/common/branches/branch-1: CHANGES.txt src/c++/task-controller/impl/configuration.c src/c++/task-controller/impl/configuration.h src/c++/task-controller/impl/main.c sr
Author: llu Date: Fri Dec 21 23:30:46 2012 New Revision: 1425195 URL: http://svn.apache.org/viewvc?rev=1425195&view=rev Log: MAPREUDCE-4397. Introduce HADOOP_SECURITY_CONF_DIR for task-controller. (Yu Gao via llu) Modified: hadoop/common/branches/branch-1/CHANGES.txt hadoop/common/branches/branch-1/src/c++/task-controller/impl/configuration.c hadoop/common/branches/branch-1/src/c++/task-controller/impl/configuration.h hadoop/common/branches/branch-1/src/c++/task-controller/impl/main.c hadoop/common/branches/branch-1/src/c++/task-controller/test/test-task-controller.c Modified: hadoop/common/branches/branch-1/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1425195&r1=1425194&r2=1425195&view=diff == --- hadoop/common/branches/branch-1/CHANGES.txt (original) +++ hadoop/common/branches/branch-1/CHANGES.txt Fri Dec 21 23:30:46 2012 @@ -6,6 +6,9 @@ Release 1.2.0 - unreleased NEW FEATURES +MAPREUDCE-4397. Introduce HADOOP_SECURITY_CONF_DIR for task-controller. +(Yu Gao via llu) + MAPREDUCE-4660. Update task placement policy for network topology with node group. (Junping Du via llu) Modified: hadoop/common/branches/branch-1/src/c++/task-controller/impl/configuration.c URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/c%2B%2B/task-controller/impl/configuration.c?rev=1425195&r1=1425194&r2=1425195&view=diff == --- hadoop/common/branches/branch-1/src/c++/task-controller/impl/configuration.c (original) +++ hadoop/common/branches/branch-1/src/c++/task-controller/impl/configuration.c Fri Dec 21 23:30:46 2012 @@ -88,6 +88,29 @@ static int is_only_root_writable(const c } /** + * Get the full path of the configuration file. + * Use $HADOOP_SECURITY_CONF_DIR for the configuration directory, and if + * it's not set, use the default value in default_conf_dir. + */ +void get_config_path(char* conf_file_path, int size, + char* default_conf_dir, + const char* conf_file_name) { + if (conf_file_name == NULL) { +fprintf(LOGFILE, "Null configuration filename passed in\n"); +exit(INVALID_CONFIG_FILE); + } + char *orig_conf_dir = getenv("HADOOP_SECURITY_CONF_DIR"); + if (orig_conf_dir == NULL) { +if (default_conf_dir == NULL) { + fprintf(LOGFILE, "Null default configuration directory passed in\n"); + exit(INVALID_CONFIG_FILE); +} +orig_conf_dir = default_conf_dir; + } + snprintf(conf_file_path, size, "%s/%s", orig_conf_dir, conf_file_name); +} + +/** * Ensure that the configuration file and all of the containing directories * are only writable by root. Otherwise, an attacker can change the * configuration and potentially cause damage. Modified: hadoop/common/branches/branch-1/src/c++/task-controller/impl/configuration.h URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/c%2B%2B/task-controller/impl/configuration.h?rev=1425195&r1=1425194&r2=1425195&view=diff == --- hadoop/common/branches/branch-1/src/c++/task-controller/impl/configuration.h (original) +++ hadoop/common/branches/branch-1/src/c++/task-controller/impl/configuration.h Fri Dec 21 23:30:46 2012 @@ -17,6 +17,15 @@ */ /** + * Get the full path of the configuration file. + * Use $HADOOP_SECURITY_CONF_DIR for the configuration directory, and if + * it's not set, use the default value in default_conf_dir. + */ +void get_config_path(char* conf_file_path, int size, + char* default_conf_dir, + const char* conf_file_name); + +/** * Ensure that the configuration file and all of the containing directories * are only writable by root. Otherwise, an attacker can change the * configuration and potentially cause damage. Modified: hadoop/common/branches/branch-1/src/c++/task-controller/impl/main.c URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/c%2B%2B/task-controller/impl/main.c?rev=1425195&r1=1425194&r2=1425195&view=diff == --- hadoop/common/branches/branch-1/src/c++/task-controller/impl/main.c (original) +++ hadoop/common/branches/branch-1/src/c++/task-controller/impl/main.c Fri Dec 21 23:30:46 2012 @@ -80,7 +80,11 @@ int main(int argc, char **argv) { #error HADOOP_CONF_DIR must be defined #endif - char *orig_conf_file = STRINGIFY(HADOOP_CONF_DIR) "/" CONF_FILENAME; + char orig_conf_file[PATH_MAX + 1]; // realpath is limitted by PATH_MAX + orig_conf_file[PATH_MAX] = 0; // in case of snprintf error + get_config_path(orig_conf_file, PATH_MAX + 1, + STRINGIFY(HADOOP_CONF_DIR), + CONF_FILENAME); char *conf_file = realpath(orig_conf_file,
svn commit: r1425193 - in /hadoop/common/branches/branch-1.1: CHANGES.txt src/c++/task-controller/impl/configuration.c src/c++/task-controller/impl/configuration.h src/c++/task-controller/impl/main.c
Author: llu Date: Fri Dec 21 23:29:30 2012 New Revision: 1425193 URL: http://svn.apache.org/viewvc?rev=1425193&view=rev Log: MAPREUDCE-4397. Introduce HADOOP_SECURITY_CONF_DIR for task-controller. (Yu Gao via llu) Modified: hadoop/common/branches/branch-1.1/CHANGES.txt hadoop/common/branches/branch-1.1/src/c++/task-controller/impl/configuration.c hadoop/common/branches/branch-1.1/src/c++/task-controller/impl/configuration.h hadoop/common/branches/branch-1.1/src/c++/task-controller/impl/main.c hadoop/common/branches/branch-1.1/src/c++/task-controller/test/test-task-controller.c Modified: hadoop/common/branches/branch-1.1/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.1/CHANGES.txt?rev=1425193&r1=1425192&r2=1425193&view=diff == --- hadoop/common/branches/branch-1.1/CHANGES.txt (original) +++ hadoop/common/branches/branch-1.1/CHANGES.txt Fri Dec 21 23:29:30 2012 @@ -6,6 +6,9 @@ Release 1.1.2 - 2012.12.07 NEW FEATURES +MAPREUDCE-4397. Introduce HADOOP_SECURITY_CONF_DIR for task-controller. +(Yu Gao via llu) + HADOOP-8561. Introduce HADOOP_PROXY_USER for secure impersonation in child hadoop client processes. (Yu Gao via llu) Modified: hadoop/common/branches/branch-1.1/src/c++/task-controller/impl/configuration.c URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.1/src/c%2B%2B/task-controller/impl/configuration.c?rev=1425193&r1=1425192&r2=1425193&view=diff == --- hadoop/common/branches/branch-1.1/src/c++/task-controller/impl/configuration.c (original) +++ hadoop/common/branches/branch-1.1/src/c++/task-controller/impl/configuration.c Fri Dec 21 23:29:30 2012 @@ -88,6 +88,29 @@ static int is_only_root_writable(const c } /** + * Get the full path of the configuration file. + * Use $HADOOP_SECURITY_CONF_DIR for the configuration directory, and if + * it's not set, use the default value in default_conf_dir. + */ +void get_config_path(char* conf_file_path, int size, + char* default_conf_dir, + const char* conf_file_name) { + if (conf_file_name == NULL) { +fprintf(LOGFILE, "Null configuration filename passed in\n"); +exit(INVALID_CONFIG_FILE); + } + char *orig_conf_dir = getenv("HADOOP_SECURITY_CONF_DIR"); + if (orig_conf_dir == NULL) { +if (default_conf_dir == NULL) { + fprintf(LOGFILE, "Null default configuration directory passed in\n"); + exit(INVALID_CONFIG_FILE); +} +orig_conf_dir = default_conf_dir; + } + snprintf(conf_file_path, size, "%s/%s", orig_conf_dir, conf_file_name); +} + +/** * Ensure that the configuration file and all of the containing directories * are only writable by root. Otherwise, an attacker can change the * configuration and potentially cause damage. Modified: hadoop/common/branches/branch-1.1/src/c++/task-controller/impl/configuration.h URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.1/src/c%2B%2B/task-controller/impl/configuration.h?rev=1425193&r1=1425192&r2=1425193&view=diff == --- hadoop/common/branches/branch-1.1/src/c++/task-controller/impl/configuration.h (original) +++ hadoop/common/branches/branch-1.1/src/c++/task-controller/impl/configuration.h Fri Dec 21 23:29:30 2012 @@ -17,6 +17,15 @@ */ /** + * Get the full path of the configuration file. + * Use $HADOOP_SECURITY_CONF_DIR for the configuration directory, and if + * it's not set, use the default value in default_conf_dir. + */ +void get_config_path(char* conf_file_path, int size, + char* default_conf_dir, + const char* conf_file_name); + +/** * Ensure that the configuration file and all of the containing directories * are only writable by root. Otherwise, an attacker can change the * configuration and potentially cause damage. Modified: hadoop/common/branches/branch-1.1/src/c++/task-controller/impl/main.c URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.1/src/c%2B%2B/task-controller/impl/main.c?rev=1425193&r1=1425192&r2=1425193&view=diff == --- hadoop/common/branches/branch-1.1/src/c++/task-controller/impl/main.c (original) +++ hadoop/common/branches/branch-1.1/src/c++/task-controller/impl/main.c Fri Dec 21 23:29:30 2012 @@ -80,7 +80,11 @@ int main(int argc, char **argv) { #error HADOOP_CONF_DIR must be defined #endif - char *orig_conf_file = STRINGIFY(HADOOP_CONF_DIR) "/" CONF_FILENAME; + char orig_conf_file[PATH_MAX + 1]; // realpath is limitted by PATH_MAX + orig_conf_file[PATH_MAX] = 0; // in case of snprintf error + get_config_path(orig_conf_file, PATH_MAX + 1, + STRINGIFY(HADOOP_CONF_DIR), + CONF
svn commit: r1425187 - in /hadoop/common/branches/branch-1: CHANGES.txt src/mapred/org/apache/hadoop/mapred/SortedRanges.java
Author: llu Date: Fri Dec 21 23:20:28 2012 New Revision: 1425187 URL: http://svn.apache.org/viewvc?rev=1425187&view=rev Log: MAPREDUCE-4272. SortedRanges.Range#compareTo is not spec compliant. (Yu Gao via llu) Modified: hadoop/common/branches/branch-1/CHANGES.txt hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/SortedRanges.java Modified: hadoop/common/branches/branch-1/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1425187&r1=1425186&r2=1425187&view=diff == --- hadoop/common/branches/branch-1/CHANGES.txt (original) +++ hadoop/common/branches/branch-1/CHANGES.txt Fri Dec 21 23:20:28 2012 @@ -161,6 +161,9 @@ Release 1.2.0 - unreleased BUG FIXES +MAPREDUCE-4272. SortedRanges.Range#compareTo is not spec compliant. +(Yu Gao via llu) + HADOOP-9051 Fix ant clean/test with circular symlinks in build dir. (llu) MAPREDUCE-4396. Port support private distributed cache to Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/SortedRanges.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/SortedRanges.java?rev=1425187&r1=1425186&r2=1425187&view=diff == --- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/SortedRanges.java (original) +++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/SortedRanges.java Fri Dec 21 23:20:28 2012 @@ -271,7 +271,7 @@ class SortedRanges implements Writable{ } public boolean equals(Object o) { - if(o!=null && o instanceof Range) { + if (o instanceof Range) { Range range = (Range)o; return startIndex==range.startIndex && length==range.length; @@ -285,10 +285,11 @@ class SortedRanges implements Writable{ } public int compareTo(Range o) { - if(this.equals(o)) { -return 0; - } - return (this.startIndex > o.startIndex) ? 1:-1; + // Ensure sgn(x.compareTo(y) == -sgn(y.compareTo(x)) + return this.startIndex < o.startIndex ? -1 : + (this.startIndex > o.startIndex ? 1 : + (this.length < o.length ? -1 : + (this.length > o.length ? 1 : 0))); } public void readFields(DataInput in) throws IOException {
svn commit: r1425185 - in /hadoop/common/branches/branch-1.1: CHANGES.txt src/mapred/org/apache/hadoop/mapred/SortedRanges.java
Author: llu Date: Fri Dec 21 23:19:09 2012 New Revision: 1425185 URL: http://svn.apache.org/viewvc?rev=1425185&view=rev Log: MAPREDUCE-4272. SortedRanges.Range#compareTo is not spec compliant. (Yu Gao via llu) Modified: hadoop/common/branches/branch-1.1/CHANGES.txt hadoop/common/branches/branch-1.1/src/mapred/org/apache/hadoop/mapred/SortedRanges.java Modified: hadoop/common/branches/branch-1.1/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.1/CHANGES.txt?rev=1425185&r1=1425184&r2=1425185&view=diff == --- hadoop/common/branches/branch-1.1/CHANGES.txt (original) +++ hadoop/common/branches/branch-1.1/CHANGES.txt Fri Dec 21 23:19:09 2012 @@ -19,6 +19,9 @@ Release 1.1.2 - 2012.12.07 BUG FIXES +MAPREDUCE-4272. SortedRanges.Range#compareTo is not spec compliant. +(Yu Gao via llu) + HADOOP-9051 Fix ant clean/test with circular symlinks in build dir. (llu) MAPREDUCE-4396. Port support private distributed cache to Modified: hadoop/common/branches/branch-1.1/src/mapred/org/apache/hadoop/mapred/SortedRanges.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.1/src/mapred/org/apache/hadoop/mapred/SortedRanges.java?rev=1425185&r1=1425184&r2=1425185&view=diff == --- hadoop/common/branches/branch-1.1/src/mapred/org/apache/hadoop/mapred/SortedRanges.java (original) +++ hadoop/common/branches/branch-1.1/src/mapred/org/apache/hadoop/mapred/SortedRanges.java Fri Dec 21 23:19:09 2012 @@ -271,7 +271,7 @@ class SortedRanges implements Writable{ } public boolean equals(Object o) { - if(o!=null && o instanceof Range) { + if (o instanceof Range) { Range range = (Range)o; return startIndex==range.startIndex && length==range.length; @@ -285,10 +285,11 @@ class SortedRanges implements Writable{ } public int compareTo(Range o) { - if(this.equals(o)) { -return 0; - } - return (this.startIndex > o.startIndex) ? 1:-1; + // Ensure sgn(x.compareTo(y) == -sgn(y.compareTo(x)) + return this.startIndex < o.startIndex ? -1 : + (this.startIndex > o.startIndex ? 1 : + (this.length < o.length ? -1 : + (this.length > o.length ? 1 : 0))); } public void readFields(DataInput in) throws IOException {
svn commit: r1425183 - in /hadoop/common/branches/branch-1.0: CHANGES.txt src/mapred/org/apache/hadoop/mapred/SortedRanges.java
Author: llu Date: Fri Dec 21 23:17:46 2012 New Revision: 1425183 URL: http://svn.apache.org/viewvc?rev=1425183&view=rev Log: MAPREDUCE-4272. SortedRanges.Range#compareTo is not spec compliant. (Yu Gao via llu) Modified: hadoop/common/branches/branch-1.0/CHANGES.txt hadoop/common/branches/branch-1.0/src/mapred/org/apache/hadoop/mapred/SortedRanges.java Modified: hadoop/common/branches/branch-1.0/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/CHANGES.txt?rev=1425183&r1=1425182&r2=1425183&view=diff == --- hadoop/common/branches/branch-1.0/CHANGES.txt (original) +++ hadoop/common/branches/branch-1.0/CHANGES.txt Fri Dec 21 23:17:46 2012 @@ -8,6 +8,9 @@ Release 1.0.5 - unreleased BUG FIXES +MAPREDUCE-4272. SortedRanges.Range#compareTo is not spec compliant. +(Yu Gao via llu) + HADOOP-9051. Fix ant clean/test with circular symlinks in build dir. (llu) MAPREDUCE-4396. Port support private distributed cache to Modified: hadoop/common/branches/branch-1.0/src/mapred/org/apache/hadoop/mapred/SortedRanges.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/src/mapred/org/apache/hadoop/mapred/SortedRanges.java?rev=1425183&r1=1425182&r2=1425183&view=diff == --- hadoop/common/branches/branch-1.0/src/mapred/org/apache/hadoop/mapred/SortedRanges.java (original) +++ hadoop/common/branches/branch-1.0/src/mapred/org/apache/hadoop/mapred/SortedRanges.java Fri Dec 21 23:17:46 2012 @@ -271,7 +271,7 @@ class SortedRanges implements Writable{ } public boolean equals(Object o) { - if(o!=null && o instanceof Range) { + if (o instanceof Range) { Range range = (Range)o; return startIndex==range.startIndex && length==range.length; @@ -285,10 +285,11 @@ class SortedRanges implements Writable{ } public int compareTo(Range o) { - if(this.equals(o)) { -return 0; - } - return (this.startIndex > o.startIndex) ? 1:-1; + // Ensure sgn(x.compareTo(y) == -sgn(y.compareTo(x)) + return this.startIndex < o.startIndex ? -1 : + (this.startIndex > o.startIndex ? 1 : + (this.length < o.length ? -1 : + (this.length > o.length ? 1 : 0))); } public void readFields(DataInput in) throws IOException {
svn commit: r1425182 - /hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
Author: jlowe Date: Fri Dec 21 23:13:35 2012 New Revision: 1425182 URL: http://svn.apache.org/viewvc?rev=1425182&view=rev Log: svn merge -c 1425177 FIXES: MAPREDUCE-4793. Problem with adding resources when using both -files and -file to hadoop streaming. Contributed by Jason Lowe Modified: hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java Modified: hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java?rev=1425182&r1=1425181&r2=1425182&view=diff == --- hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java Fri Dec 21 23:13:35 2012 @@ -306,7 +306,13 @@ public class StreamJob implements Tool { throw new IllegalArgumentException(e); } } -config_.set("tmpfiles", config_.get("tmpfiles", "") + fileList); +String tmpFiles = config_.get("tmpfiles", ""); +if (tmpFiles.isEmpty()) { + tmpFiles = fileList.toString(); +} else { + tmpFiles = tmpFiles + "," + fileList; +} +config_.set("tmpfiles", tmpFiles); validate(packageFiles_); }
svn commit: r1425181 - /hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
Author: jlowe Date: Fri Dec 21 23:12:44 2012 New Revision: 1425181 URL: http://svn.apache.org/viewvc?rev=1425181&view=rev Log: svn merge -c 1425177 FIXES: MAPREDUCE-4793. Problem with adding resources when using both -files and -file to hadoop streaming. Contributed by Jason Lowe Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java?rev=1425181&r1=1425180&r2=1425181&view=diff == --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java (original) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java Fri Dec 21 23:12:44 2012 @@ -306,7 +306,13 @@ public class StreamJob implements Tool { throw new IllegalArgumentException(e); } } -config_.set("tmpfiles", config_.get("tmpfiles", "") + fileList); +String tmpFiles = config_.get("tmpfiles", ""); +if (tmpFiles.isEmpty()) { + tmpFiles = fileList.toString(); +} else { + tmpFiles = tmpFiles + "," + fileList; +} +config_.set("tmpfiles", tmpFiles); validate(packageFiles_); }
svn commit: r1425177 - /hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
Author: jlowe Date: Fri Dec 21 23:05:54 2012 New Revision: 1425177 URL: http://svn.apache.org/viewvc?rev=1425177&view=rev Log: MAPREDUCE-4793. Problem with adding resources when using both -files and -file to hadoop streaming. Contributed by Jason Lowe Modified: hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java Modified: hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java?rev=1425177&r1=1425176&r2=1425177&view=diff == --- hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java (original) +++ hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java Fri Dec 21 23:05:54 2012 @@ -306,7 +306,13 @@ public class StreamJob implements Tool { throw new IllegalArgumentException(e); } } -config_.set("tmpfiles", config_.get("tmpfiles", "") + fileList); +String tmpFiles = config_.get("tmpfiles", ""); +if (tmpFiles.isEmpty()) { + tmpFiles = fileList.toString(); +} else { + tmpFiles = tmpFiles + "," + fileList; +} +config_.set("tmpfiles", tmpFiles); validate(packageFiles_); }
svn commit: r1425171 - in /hadoop/common/branches/branch-1: ./ src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/ src/mapred/ src/mapred/org/apache/hadoop/mapred/ src/test/org/apache/hadoop/
Author: llu Date: Fri Dec 21 22:40:37 2012 New Revision: 1425171 URL: http://svn.apache.org/viewvc?rev=1425171&view=rev Log: MAPREDUCE-4660. Update task placement policy for network topology with node group. (Junping Du via llu) Added: hadoop/common/branches/branch-1/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/JobSchedulableWithNodeGroup.java hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/MiniMRClusterWithNodeGroup.java hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestNodeGroupAwareTaskPlacement.java Modified: hadoop/common/branches/branch-1/CHANGES.txt hadoop/common/branches/branch-1/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java hadoop/common/branches/branch-1/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/JobSchedulable.java hadoop/common/branches/branch-1/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/LocalityLevel.java hadoop/common/branches/branch-1/src/mapred/mapred-default.xml hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress.java hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress_Counter.properties hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobQueueTaskScheduler.java hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobTracker.java hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/MiniMRCluster.java Modified: hadoop/common/branches/branch-1/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1425171&r1=1425170&r2=1425171&view=diff == --- hadoop/common/branches/branch-1/CHANGES.txt (original) +++ hadoop/common/branches/branch-1/CHANGES.txt Fri Dec 21 22:40:37 2012 @@ -6,6 +6,9 @@ Release 1.2.0 - unreleased NEW FEATURES +MAPREDUCE-4660. Update task placement policy for network topology +with node group. (Junping Du via llu) + HADOOP-8561. Introduce HADOOP_PROXY_USER for secure impersonation in child hadoop client processes. (Yu Gao via llu) Modified: hadoop/common/branches/branch-1/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java?rev=1425171&r1=1425170&r2=1425171&view=diff == --- hadoop/common/branches/branch-1/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java (original) +++ hadoop/common/branches/branch-1/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java Fri Dec 21 22:40:37 2012 @@ -41,6 +41,7 @@ import org.apache.hadoop.mapreduce.serve import org.apache.hadoop.metrics.MetricsContext; import org.apache.hadoop.metrics.MetricsUtil; import org.apache.hadoop.metrics.Updater; +import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.StringUtils; @@ -327,8 +328,17 @@ public class FairScheduler extends TaskS public void jobAdded(JobInProgress job) { synchronized (FairScheduler.this) { eventLog.log("JOB_ADDED", job.getJobID()); -JobInfo info = new JobInfo(new JobSchedulable(FairScheduler.this, job, TaskType.MAP), -new JobSchedulable(FairScheduler.this, job, TaskType.REDUCE)); +JobSchedulable mapSched = ReflectionUtils.newInstance( +conf.getClass("mapred.jobtracker.jobSchedulable", JobSchedulable.class, +JobSchedulable.class), conf); +mapSched.init(FairScheduler.this, job, TaskType.MAP); + +JobSchedulable redSched = ReflectionUtils.newInstance( +conf.getClass("mapred.jobtracker.jobSchedulable", JobSchedulable.class, +JobSchedulable.class), conf); +redSched.init(FairScheduler.this, job, TaskType.REDUCE); + +JobInfo info = new JobInfo(mapSched, redSched); infos.put(job, info); poolMgr.addJob(job); // Also adds job into the right PoolScheduable update(); @@ -585,8 +595,10 @@ public class FairScheduler extends TaskS private void updateLastMapLocalityLevel(JobInProgress job, Task mapTaskLaunched, TaskTrackerStatus tracker) { JobInfo info = infos.get(job); +boolean isNodeGroupAware = conf.getBoolean( +"net.topology.nodegroup.aware", false); LocalityLevel localityLevel = LocalityLevel.fromTask( -job, mapTaskLaunched, tracker); +job, mapTaskLaunched, tracker, isNodeGroupAware); info.lastMapLocalityLevel = localityLevel; info.timeWaitedForLocalMap = 0; eventLog.log("ASSIGNED_LOC_LEVEL", job.getJobID(), localityLevel); Modified: hadoop/common/branches/branch-1/src/con
svn commit: r1425095 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/fs/DU.java src/test/java/org/apache/hadoop/fs/TestDU.java
Author: tgraves Date: Fri Dec 21 19:06:39 2012 New Revision: 1425095 URL: http://svn.apache.org/viewvc?rev=1425095&view=rev Log: HADOOP-9152. HDFS can report negative DFS Used on clusters with very small amounts of data (Brock Noland via tgraves) Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDU.java Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1425095&r1=1425094&r2=1425095&view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Fri Dec 21 19:06:39 2012 @@ -39,6 +39,9 @@ Release 0.23.6 - UNRELEASED HADOOP-9105. FsShell -moveFromLocal erroneously fails (daryn via bobby) +HADOOP-9152. HDFS can report negative DFS Used on clusters with very +small amounts of data (Brock Noland via tgraves) + Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java?rev=1425095&r1=1425094&r2=1425095&view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java Fri Dec 21 19:06:39 2012 @@ -135,7 +135,7 @@ public class DU extends Shell { } } -return used.longValue(); +return Math.max(used.longValue(), 0L); } /** Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDU.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDU.java?rev=1425095&r1=1425094&r2=1425095&view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDU.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDU.java Fri Dec 21 19:06:39 2012 @@ -101,4 +101,12 @@ public class TestDU extends TestCase { duSize >= writtenSize && writtenSize <= (duSize + slack)); } + public void testDUGetUsedWillNotReturnNegative() throws IOException { +File file = new File(DU_DIR, "data"); +assertTrue(file.createNewFile()); +DU du = new DU(file, 1); +du.decDfsUsed(Long.MAX_VALUE); +long duSize = du.getUsed(); +assertTrue(String.valueOf(duSize), duSize >= 0L); + } }
[Hadoop Wiki] Trivial Update of "FrontPage" by SimoneLeo
Dear Wiki user, You have subscribed to a wiki page or wiki category on "Hadoop Wiki" for change notification. The "FrontPage" page has been changed by SimoneLeo: http://wiki.apache.org/hadoop/FrontPage?action=diff&rev1=297&rev2=298 Comment: Updated info on Pydoop * [[http://www.wheregridenginelives.com/content/big-data-big-compute-grid-engine-and-hadoop-0|Univa Grid Engine Integration]] A blog post about the integration of Hadoop with the Grid Engine successor Univa Grid Engine * [[http://gridscheduler.sourceforge.net/howto/GridEngineHadoop.html|Hadoop Grid Engine Integration]] Open Grid Scheduler/Grid Engine Hadoop integration setup instructions. * [[http://philippeadjiman.com/blog/the-hadoop-tutorial-series/|Hadoop Tutorial Series]] Learning progressively important core Hadoop concepts with hands-on experiments using the Cloudera Virtual Machine - * [[http://pydoop.sourceforge.net|Pydoop]] A Python MapReduce and HDFS API for Hadoop. + * [[http://pydoop.sourceforge.net|Pydoop]] A Python !MapReduce and HDFS API for Hadoop ([[http://pydoop.sourceforge.net/docs/tutorial|tutorial]]). * [[https://github.com/klbostee/dumbo/wiki|Dumbo]] Dumbo is a project that allows you to easily write and run Hadoop programs in Python. * [[http://www.asterdata.com/news/091001-Aster-Hadoop-connector.php|Hadoop distributed file system]] New Hadoop Connector Enables Ultra-Fast Transfer of Data between Hadoop and Aster Data's MPP Data Warehouse. * [[CUDA On Hadoop|Hadoop + CUDA]]