svn commit: r1588470 - /hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java
Author: jlowe
Date: Fri Apr 18 13:40:19 2014
New Revision: 1588470
URL: http://svn.apache.org/r1588470
Log:
svn merge -c 1451291 FIXES: HADOOP-9230. TestUniformSizeInputFormat fails
intermittently. Contributed by Karthik Kambatla
Modified:
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java?rev=1588470r1=1588469r2=1588470view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java
Fri Apr 18 13:40:19 2014
@@ -33,8 +33,6 @@ import org.apache.hadoop.tools.CopyListi
import org.apache.hadoop.tools.DistCpOptions;
import org.apache.hadoop.tools.StubContext;
import org.apache.hadoop.security.Credentials;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -48,9 +46,6 @@ import java.util.Random;
public class TestUniformSizeInputFormat {
- private static final Log LOG
-= LogFactory.getLog(TestUniformSizeInputFormat.class);
-
private static MiniDFSCluster cluster;
private static final int N_FILES = 20;
private static final int SIZEOF_EACH_FILE=1024;
@@ -118,12 +113,9 @@ public class TestUniformSizeInputFormat
ListInputSplit splits
= uniformSizeInputFormat.getSplits(jobContext);
-ListInputSplit legacySplits = legacyGetSplits(listFile, nMaps);
-
int sizePerMap = totalFileSize/nMaps;
checkSplits(listFile, splits);
-checkAgainstLegacy(splits, legacySplits);
int doubleCheckedTotalSize = 0;
int previousSplitSize = -1;
@@ -155,57 +147,6 @@ public class TestUniformSizeInputFormat
Assert.assertEquals(totalFileSize, doubleCheckedTotalSize);
}
- // From
- //
http://svn.apache.org/repos/asf/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/DistCp.java
- private ListInputSplit legacyGetSplits(Path listFile, int numSplits)
- throws IOException {
-
-FileSystem fs = cluster.getFileSystem();
-FileStatus srcst = fs.getFileStatus(listFile);
-Configuration conf = fs.getConf();
-
-ArrayListInputSplit splits = new ArrayListInputSplit(numSplits);
-FileStatus value = new FileStatus();
-Text key = new Text();
-final long targetsize = totalFileSize / numSplits;
-long pos = 0L;
-long last = 0L;
-long acc = 0L;
-long cbrem = srcst.getLen();
-SequenceFile.Reader sl = null;
-
-LOG.info(Average bytes per map: + targetsize +
-, Number of maps: + numSplits + , total size: + totalFileSize);
-
-try {
- sl = new SequenceFile.Reader(conf, SequenceFile.Reader.file(listFile));
- for (; sl.next(key, value); last = sl.getPosition()) {
-// if adding this split would put this split past the target size,
-// cut the last split and put this next file in the next split.
-if (acc + value.getLen() targetsize acc != 0) {
- long splitsize = last - pos;
- FileSplit fileSplit = new FileSplit(listFile, pos, splitsize, null);
- LOG.info (Creating split : + fileSplit + , bytes in split: +
splitsize);
- splits.add(fileSplit);
- cbrem -= splitsize;
- pos = last;
- acc = 0L;
-}
-acc += value.getLen();
- }
-}
-finally {
- IOUtils.closeStream(sl);
-}
-if (cbrem != 0) {
- FileSplit fileSplit = new FileSplit(listFile, pos, cbrem, null);
- LOG.info (Creating split : + fileSplit + , bytes in split: +
cbrem);
- splits.add(fileSplit);
-}
-
-return splits;
- }
-
private void checkSplits(Path listFile, ListInputSplit splits) throws
IOException {
long lastEnd = 0;
@@ -233,18 +174,6 @@ public class TestUniformSizeInputFormat
}
}
- private void checkAgainstLegacy(ListInputSplit splits,
- ListInputSplit legacySplits)
- throws IOException, InterruptedException {
-
-Assert.assertEquals(legacySplits.size(), splits.size());
-for (int index = 0; index splits.size(); index++) {
- FileSplit fileSplit = (FileSplit) splits.get(index);
- FileSplit legacyFileSplit = (FileSplit) legacySplits.get(index);
- Assert.assertEquals(fileSplit.getStart(), legacyFileSplit.getStart());
-}
- }
-
@Test
public void
svn commit: r1588470 - /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Author: jlowe Date: Fri Apr 18 13:40:19 2014 New Revision: 1588470 URL: http://svn.apache.org/r1588470 Log: svn merge -c 1451291 FIXES: HADOOP-9230. TestUniformSizeInputFormat fails intermittently. Contributed by Karthik Kambatla Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1588470r1=1588469r2=1588470view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Fri Apr 18 13:40:19 2014 @@ -35,6 +35,9 @@ Release 0.23.11 - UNRELEASED HADOOP-10332. HttpServer's jetty audit log always logs 200 OK (jeagles) +HADOOP-9230. TestUniformSizeInputFormat fails intermittently. +(kkambatl via jlowe) + Release 0.23.10 - 2013-12-09 INCOMPATIBLE CHANGES
svn commit: r1588509 [1/4] - in /hadoop/common/branches/HADOOP-10388/hadoop-common-project: hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/ hadoop-auth/src/test/java/org/ap
Author: cmccabe Date: Fri Apr 18 16:32:35 2014 New Revision: 1588509 URL: http://svn.apache.org/r1588509 Log: Merge trunk into the HADOOP-10388 branch Added: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFs.java - copied unchanged from r1588387, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFs.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/test/resources/javakeystoreprovider.password - copied unchanged from r1588387, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/resources/javakeystoreprovider.password Removed: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/BinaryRecordInput.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/BinaryRecordOutput.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Buffer.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/CsvRecordInput.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/CsvRecordOutput.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Index.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Record.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/RecordComparator.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/RecordInput.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/RecordOutput.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Utils.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/XmlRecordInput.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/XmlRecordOutput.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/compiler/CGenerator.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/compiler/CodeBuffer.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/compiler/CodeGenerator.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/compiler/Consts.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/compiler/CppGenerator.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/compiler/JBoolean.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/compiler/JBuffer.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/compiler/JByte.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/compiler/JCompType.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/compiler/JDouble.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/compiler/JField.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/compiler/JFile.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/compiler/JFloat.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/compiler/JInt.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/compiler/JLong.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/compiler/JMap.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/compiler/JRecord.java hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/compiler/JString.java
svn commit: r1588578 - in /hadoop/common/branches/branch-1: CHANGES.txt src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java src/test/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.jav
Author: cnauroth
Date: Fri Apr 18 21:56:35 2014
New Revision: 1588578
URL: http://svn.apache.org/r1588578
Log:
HDFS-6141. WebHdfsFileSystem#toUrl does not perform character escaping.
Contributed by Chris Nauroth.
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1588578r1=1588577r2=1588578view=diff
==
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Fri Apr 18 21:56:35 2014
@@ -208,6 +208,9 @@ Release 1.3.0 - unreleased
MAPREDUCE-5822. FairScheduler does not preempt due to fairshare-starvation
when fairshare is 1. (Anubhav Dhoot via kasha)
+HDFS-6141. WebHdfsFileSystem#toUrl does not perform character escaping.
+(cnauroth)
+
Release 1.2.2 - unreleased
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java?rev=1588578r1=1588577r2=1588578view=diff
==
---
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
(original)
+++
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
Fri Apr 18 21:56:35 2014
@@ -373,7 +373,7 @@ public class WebHdfsFileSystem extends F
final Param?,?... parameters) throws IOException {
//initialize URI path and query
final String path = PATH_PREFIX
-+ (fspath == null? /: makeQualified(fspath).toUri().getPath());
++ (fspath == null? /: makeQualified(fspath).toUri().getRawPath());
final String query = op.toQueryString()
+ '' + new UserParam(ugi)
+ Param.toSortedString(, parameters);
Modified:
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java?rev=1588578r1=1588577r2=1588578view=diff
==
---
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
(original)
+++
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
Fri Apr 18 21:56:35 2014
@@ -383,7 +383,7 @@ public class TestWebHdfsFileSystemContra
{//test create with path containing spaces
HttpOpParam.Op op = PutOpParam.Op.CREATE;
- Path path = new Path(/test/path%20with%20spaces);
+ Path path = new Path(/test/path with spaces);
URL url = webhdfs.toUrl(op, path);
HttpURLConnection conn = (HttpURLConnection)url.openConnection();
conn.setRequestMethod(op.getType().toString());
