Author: szetszwo
Date: Tue Nov 6 21:11:42 2012
New Revision: 1406337
URL: http://svn.apache.org/viewvc?rev=1406337&view=rev
Log:
Merge r1406007 through r1406326 from trunk.
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt
(contents, props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/conf/ (props
changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedPartitioner.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestKeyFieldBasedPartitioner.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/c++/ (props
changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/block_forensics/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/build-contrib.xml
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/build.xml
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/data_join/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/eclipse-plugin/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/index/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/vaidya/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/examples/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/java/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/test/mapred/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/webapps/job/
(props changed)
Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1405252-1406326
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt?rev=1406337&r1=1406336&r2=1406337&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt Tue
Nov 6 21:11:42 2012
@@ -631,6 +631,9 @@ Release 0.23.5 - UNRELEASED
MAPREDUCE-4763 repair test TestUmbilicalProtocolWithJobToken (Ivan A.
Veselovsky via bobby)
+
+ MAPREDUCE-4771. KeyFieldBasedPartitioner not partitioning properly when
+ configured (jlowe via bobby)
Release 0.23.4 - UNRELEASED
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1405252-1406326
Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1405252-1406326
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedPartitioner.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedPartitioner.java?rev=1406337&r1=1406336&r2=1406337&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedPartitioner.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedPartitioner.java
Tue Nov 6 21:11:42 2012
@@ -63,6 +63,7 @@ public class KeyFieldBasedPartitioner<K2
public void setConf(Configuration conf) {
this.conf = conf;
+ keyFieldHelper = new KeyFieldHelper();
String keyFieldSeparator =
conf.get(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPERATOR, "\t");
keyFieldHelper.setKeyFieldSeparator(keyFieldSeparator);
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1405252-1406326
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestKeyFieldBasedPartitioner.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestKeyFieldBasedPartitioner.java?rev=1406337&r1=1406336&r2=1406337&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestKeyFieldBasedPartitioner.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestKeyFieldBasedPartitioner.java
Tue Nov 6 21:11:42 2012
@@ -17,17 +17,18 @@
*/
package org.apache.hadoop.mapred.lib;
+import static org.junit.Assert.assertEquals;
+
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner;
-
-import junit.framework.TestCase;
+import org.junit.Test;
-public class TestKeyFieldBasedPartitioner extends TestCase {
+public class TestKeyFieldBasedPartitioner {
/**
* Test is key-field-based partitioned works with empty key.
*/
+ @Test
public void testEmptyKey() throws Exception {
KeyFieldBasedPartitioner<Text, Text> kfbp =
new KeyFieldBasedPartitioner<Text, Text>();
@@ -37,4 +38,18 @@ public class TestKeyFieldBasedPartitione
assertEquals("Empty key should map to 0th partition",
0, kfbp.getPartition(new Text(), new Text(), 10));
}
+
+ @Test
+ public void testMultiConfigure() {
+ KeyFieldBasedPartitioner<Text, Text> kfbp =
+ new KeyFieldBasedPartitioner<Text, Text>();
+ JobConf conf = new JobConf();
+ conf.set(KeyFieldBasedPartitioner.PARTITIONER_OPTIONS, "-k1,1");
+ kfbp.setConf(conf);
+ Text key = new Text("foo\tbar");
+ Text val = new Text("val");
+ int partNum = kfbp.getPartition(key, val, 4096);
+ kfbp.configure(conf);
+ assertEquals(partNum, kfbp.getPartition(key,val, 4096));
+ }
}
\ No newline at end of file
Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/c++/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/c++:r1405252-1406326
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib:r1405252-1406326
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/block_forensics/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/block_forensics:r1405252-1406326
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/build-contrib.xml
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build-contrib.xml:r1405252-1406326
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/build.xml
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build.xml:r1405252-1406326
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/data_join/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/data_join:r1405252-1406326
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/eclipse-plugin/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/eclipse-plugin:r1405252-1406326
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/index/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/index:r1405252-1406326
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/vaidya/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/vaidya:r1405252-1406326
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/examples/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/examples:r1405252-1406326
Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/java:r1405252-1406326
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/test/mapred/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred:r1405252-1406326
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs:r1405252-1406326
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs:r1405252-1406326
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc:r1405252-1406326
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/webapps/job/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/src/webapps/job:r1405252-1406326
