Author: arp
Date: Wed Aug 21 17:47:10 2013
New Revision: 1516230
URL: http://svn.apache.org/r1516230
Log:
Merge all changes from trunk to branch HDFS-2832
Modified:
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt
(contents, props changed)
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
Propchange: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1513717-1516228
Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt?rev=1516230&r1=1516229&r2=1516230&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt Wed
Aug 21 17:47:10 2013
@@ -162,9 +162,6 @@ Release 2.3.0 - UNRELEASED
OPTIMIZATIONS
- MAPREDUCE-1981. Improve getSplits performance by using listLocatedStatus
- (Hairong Kuang and Jason Lowe via jlowe)
-
BUG FIXES
MAPREDUCE-5316. job -list-attempt-ids command does not handle illegal
@@ -186,12 +183,15 @@ Release 2.1.1-beta - UNRELEASED
OPTIMIZATIONS
- MAPREDUCE-5352. Optimize node local splits generated by
- CombineFileInputFormat. (sseth)
-
MAPREDUCE-5446. TestJobHistoryEvents and TestJobHistoryParsing have race
conditions (jlowe via kihwal)
+ MAPREDUCE-5462. In map-side sort, swap entire meta entries instead of
+ indexes for better cache performance. (Sandy Ryza)
+
+ MAPREDUCE-1981. Improve getSplits performance by using listLocatedStatus
+ (Hairong Kuang and Jason Lowe via jlowe)
+
BUG FIXES
MAPREDUCE-5385. Fixed a bug with JobContext getCacheFiles API. (Omkar Vinit
@@ -223,7 +223,10 @@ Release 2.1.1-beta - UNRELEASED
MAPREDUCE-5454. TestDFSIO fails intermittently on JDK7 (Karthik Kambatla
via Sandy Ryza)
-Release 2.1.0-beta - 2013-08-06
+ MAPREDUCE-5001. LocalJobRunner has race condition resulting in job
+ failures (Sandy Ryza via jlowe)
+
+Release 2.1.0-beta - 2013-08-22
INCOMPATIBLE CHANGES
@@ -395,6 +398,9 @@ Release 2.1.0-beta - 2013-08-06
MAPREDUCE-5268. Improve history server startup performance (Karthik
Kambatla via jlowe)
+ MAPREDUCE-5352. Optimize node local splits generated by
+ CombineFileInputFormat. (sseth)
+
BUG FIXES
MAPREDUCE-4671. AM does not tell the RM about container requests which are
@@ -1295,6 +1301,9 @@ Release 0.23.10 - UNRELEASED
MAPREDUCE-5440. TestCopyCommitter Fails on JDK7 (Robert Parker via jlowe)
+ MAPREDUCE-5001. LocalJobRunner has race condition resulting in job
+ failures (Sandy Ryza via jlowe)
+
Release 0.23.9 - 2013-07-08
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1513717-1516228
Modified:
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml?rev=1516230&r1=1516229&r2=1516230&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
Wed Aug 21 17:47:10 2013
@@ -64,7 +64,8 @@
<goal>protoc</goal>
</goals>
<configuration>
- <protocVersion>2.5.0</protocVersion>
+ <protocVersion>${protobuf.version}</protocVersion>
+ <protocCommand>${protoc.path}</protocCommand>
<imports>
<param>${basedir}/../../../hadoop-common-project/hadoop-common/src/main/proto</param>
<param>${basedir}/../../../hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto</param>
Modified:
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java?rev=1516230&r1=1516229&r2=1516230&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
Wed Aug 21 17:47:10 2013
@@ -884,10 +884,10 @@ public class MapTask extends Task {
byte[] kvbuffer; // main output buffer
private final byte[] b0 = new byte[0];
- private static final int INDEX = 0; // index offset in acct
- private static final int VALSTART = 1; // val offset in acct
- private static final int KEYSTART = 2; // key offset in acct
- private static final int PARTITION = 3; // partition offset in acct
+ private static final int VALSTART = 0; // val offset in acct
+ private static final int KEYSTART = 1; // key offset in acct
+ private static final int PARTITION = 2; // partition offset in acct
+ private static final int VALLEN = 3; // length of value
private static final int NMETA = 4; // num meta ints
private static final int METASIZE = NMETA * 4; // size in bytes
@@ -1151,10 +1151,10 @@ public class MapTask extends Task {
distanceTo(keystart, valend, bufvoid));
// write accounting info
- kvmeta.put(kvindex + INDEX, kvindex);
kvmeta.put(kvindex + PARTITION, partition);
kvmeta.put(kvindex + KEYSTART, keystart);
kvmeta.put(kvindex + VALSTART, valstart);
+ kvmeta.put(kvindex + VALLEN, distanceTo(valstart, valend));
// advance kvindex
kvindex = (kvindex - NMETA + kvmeta.capacity()) % kvmeta.capacity();
} catch (MapBufferTooSmallException e) {
@@ -1224,17 +1224,11 @@ public class MapTask extends Task {
}
/**
- * For the given meta position, return the dereferenced position in the
- * integer array. Each meta block contains several integers describing
- * record data in its serialized form, but the INDEX is not necessarily
- * related to the proximate metadata. The index value at the referenced int
- * position is the start offset of the associated metadata block. So the
- * metadata INDEX at metapos may point to the metadata described by the
- * metadata block at metapos + k, which contains information about that
- * serialized record.
+ * For the given meta position, return the offset into the int-sized
+ * kvmeta buffer.
*/
int offsetFor(int metapos) {
- return kvmeta.get(metapos * NMETA + INDEX);
+ return metapos * NMETA;
}
/**
@@ -1260,16 +1254,17 @@ public class MapTask extends Task {
kvmeta.get(kvj + VALSTART) - kvmeta.get(kvj + KEYSTART));
}
+ final byte META_BUFFER_TMP[] = new byte[METASIZE];
/**
- * Swap logical indices st i, j MOD offset capacity.
+ * Swap metadata for items i, j
* @see IndexedSortable#swap
*/
public void swap(final int mi, final int mj) {
- final int kvi = (mi % maxRec) * NMETA + INDEX;
- final int kvj = (mj % maxRec) * NMETA + INDEX;
- int tmp = kvmeta.get(kvi);
- kvmeta.put(kvi, kvmeta.get(kvj));
- kvmeta.put(kvj, tmp);
+ int iOff = (mi % maxRec) * METASIZE;
+ int jOff = (mj % maxRec) * METASIZE;
+ System.arraycopy(kvbuffer, iOff, META_BUFFER_TMP, 0, METASIZE);
+ System.arraycopy(kvbuffer, jOff, kvbuffer, iOff, METASIZE);
+ System.arraycopy(META_BUFFER_TMP, 0, kvbuffer, jOff, METASIZE);
}
/**
@@ -1601,9 +1596,9 @@ public class MapTask extends Task {
while (spindex < mend &&
kvmeta.get(offsetFor(spindex % maxRec) + PARTITION) == i) {
final int kvoff = offsetFor(spindex % maxRec);
- key.reset(kvbuffer, kvmeta.get(kvoff + KEYSTART),
- (kvmeta.get(kvoff + VALSTART) -
- kvmeta.get(kvoff + KEYSTART)));
+ int keystart = kvmeta.get(kvoff + KEYSTART);
+ int valstart = kvmeta.get(kvoff + VALSTART);
+ key.reset(kvbuffer, keystart, valstart - keystart);
getVBytesForOffset(kvoff, value);
writer.append(key, value);
++spindex;
@@ -1729,14 +1724,8 @@ public class MapTask extends Task {
private void getVBytesForOffset(int kvoff, InMemValBytes vbytes) {
// get the keystart for the next serialized value to be the end
// of this value. If this is the last value in the buffer, use bufend
- final int nextindex = kvoff == kvend
- ? bufend
- : kvmeta.get(
- (kvoff - NMETA + kvmeta.capacity() + KEYSTART) %
kvmeta.capacity());
- // calculate the length of the value
- int vallen = (nextindex >= kvmeta.get(kvoff + VALSTART))
- ? nextindex - kvmeta.get(kvoff + VALSTART)
- : (bufvoid - kvmeta.get(kvoff + VALSTART)) + nextindex;
+ final int vallen = kvmeta.get(kvoff + VALLEN);
+ assert vallen >= 0;
vbytes.reset(kvbuffer, kvmeta.get(kvoff + VALSTART), vallen);
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java?rev=1516230&r1=1516229&r2=1516230&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
Wed Aug 21 17:47:10 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.mapreduce;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
@@ -181,7 +182,18 @@ public class Cluster {
public Job getJob(JobID jobId) throws IOException, InterruptedException {
JobStatus status = client.getJobStatus(jobId);
if (status != null) {
- return Job.getInstance(this, status, new JobConf(status.getJobFile()));
+ JobConf conf;
+ try {
+ conf = new JobConf(status.getJobFile());
+ } catch (RuntimeException ex) {
+ // If job file doesn't exist it means we can't find the job
+ if (ex.getCause() instanceof FileNotFoundException) {
+ return null;
+ } else {
+ throw ex;
+ }
+ }
+ return Job.getInstance(this, status, conf);
}
return null;
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java?rev=1516230&r1=1516229&r2=1516230&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java
Wed Aug 21 17:47:10 2013
@@ -102,7 +102,8 @@ public class TestJobEndNotifier extends
public void setUp() throws Exception {
new File(System.getProperty("build.webapps", "build/webapps") + "/test"
).mkdirs();
- server = new HttpServer("test", "0.0.0.0", 0, true);
+ server = new HttpServer.Builder().setName("test")
+ .setBindAddress("0.0.0.0").setPort(0).setFindPort(true).build();
server.addServlet("delay", "/delay", DelayServlet.class);
server.addServlet("jobend", "/jobend", JobEndServlet.class);
server.addServlet("fail", "/fail", FailServlet.class);
Modified:
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml?rev=1516230&r1=1516229&r2=1516230&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
Wed Aug 21 17:47:10 2013
@@ -78,7 +78,8 @@
<goal>protoc</goal>
</goals>
<configuration>
- <protocVersion>2.5.0</protocVersion>
+ <protocVersion>${protobuf.version}</protocVersion>
+ <protocCommand>${protoc.path}</protocCommand>
<imports>
<param>
${basedir}/../../../../hadoop-common-project/hadoop-common/src/main/proto
