git commit: HDFS Credential Provider related Unit Test Failure. Contributed by Xiaoyu Yao.
Repository: hadoop
Updated Branches:
refs/heads/trunk d1fa58292 -> a23144fd8
HDFS Credential Provider related Unit Test Failure. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a23144fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a23144fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a23144fd
Branch: refs/heads/trunk
Commit: a23144fd8a1e399e431f2f272388ec109df37ab1
Parents: d1fa582
Author: cnauroth
Authored: Sun Sep 7 08:39:20 2014 -0700
Committer: cnauroth
Committed: Sun Sep 7 08:39:20 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/cli/TestCryptoAdminCLI.java | 4 +++-
.../test/java/org/apache/hadoop/hdfs/TestDFSUtil.java | 4 +++-
.../org/apache/hadoop/hdfs/TestEncryptionZones.java | 12 +++-
.../org/apache/hadoop/hdfs/TestReservedRawPaths.java| 3 ++-
5 files changed, 18 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23144fd/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 3d43171..5a30d0a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -615,6 +615,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6898. DN must reserve space for a full block when an RBW block is
created. (Arpit Agarwal)
+HDFS-7025. HDFS Credential Provider related Unit Test Failure.
+(Xiaoyu Yao via cnauroth)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23144fd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
index 1c83829..adeabfe 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderFactory;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HDFSPolicyProvider;
@@ -64,8 +65,9 @@ public class TestCryptoAdminCLI extends CLITestHelperDFS {
tmpDir = new File(System.getProperty("test.build.data", "target"),
UUID.randomUUID().toString()).getAbsoluteFile();
+final Path jksPath = new Path(tmpDir.toString(), "test.jks");
conf.set(KeyProviderFactory.KEY_PROVIDER_PATH,
-JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks");
+JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri());
dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
dfsCluster.waitClusterUp();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23144fd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
index 5ffd3b5..046265f 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@@ -809,8 +810,9 @@ public class TestDFSUtil {
"target/test-dir"));
Configuration conf = new Configuration();
+final Path jksPath = new Path(testDir.toString(), "test.jks");
final String ourUrl =
-JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks";
+JavaKeyStoreProvider.
git commit: HDFS Credential Provider related Unit Test Failure. Contributed by Xiaoyu Yao.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 eba06e61c -> b87d1d7d0
HDFS Credential Provider related Unit Test Failure. Contributed by Xiaoyu Yao.
(cherry picked from commit a23144fd8a1e399e431f2f272388ec109df37ab1)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b87d1d7d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b87d1d7d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b87d1d7d
Branch: refs/heads/branch-2
Commit: b87d1d7d0abd73da2a2bf548404a517a53ac9eb0
Parents: eba06e6
Author: cnauroth
Authored: Sun Sep 7 08:39:20 2014 -0700
Committer: cnauroth
Committed: Sun Sep 7 08:41:08 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/cli/TestCryptoAdminCLI.java | 4 +++-
.../test/java/org/apache/hadoop/hdfs/TestDFSUtil.java | 4 +++-
.../org/apache/hadoop/hdfs/TestEncryptionZones.java | 12 +++-
.../org/apache/hadoop/hdfs/TestReservedRawPaths.java| 3 ++-
5 files changed, 18 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b87d1d7d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0393164..d31cb26 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -354,6 +354,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6898. DN must reserve space for a full block when an RBW block is
created. (Arpit Agarwal)
+HDFS-7025. HDFS Credential Provider related Unit Test Failure.
+(Xiaoyu Yao via cnauroth)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b87d1d7d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
index 1c83829..adeabfe 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderFactory;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HDFSPolicyProvider;
@@ -64,8 +65,9 @@ public class TestCryptoAdminCLI extends CLITestHelperDFS {
tmpDir = new File(System.getProperty("test.build.data", "target"),
UUID.randomUUID().toString()).getAbsoluteFile();
+final Path jksPath = new Path(tmpDir.toString(), "test.jks");
conf.set(KeyProviderFactory.KEY_PROVIDER_PATH,
-JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks");
+JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri());
dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
dfsCluster.waitClusterUp();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b87d1d7d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
index 5ffd3b5..046265f 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@@ -809,8 +810,9 @@ public class TestDFSUtil {
"target/test-dir"));
Configuration conf = new Configuration();
+final Path jksPath = new Path(testDir.toString(), "test.jks");
final String ourUrl =
-JavaKeyStoreProvider.
[2/4] git commit: HDFS-6898. DN must reserve space for a full block when an RBW block is created. (Contributed by Arpit Agarwal)
HDFS-6898. DN must reserve space for a full block when an RBW block is created.
(Contributed by Arpit Agarwal)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d1fa5829
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d1fa5829
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d1fa5829
Branch: refs/heads/HDFS-6581
Commit: d1fa58292e87bc29b4ef1278368c2be938a0afc4
Parents: cbea1b1
Author: arp
Authored: Sat Sep 6 20:02:40 2014 -0700
Committer: arp
Committed: Sat Sep 6 21:04:29 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hadoop/hdfs/protocol/HdfsConstants.java | 2 +-
.../server/datanode/ReplicaBeingWritten.java| 12 +-
.../hdfs/server/datanode/ReplicaInPipeline.java | 33 ++-
.../hdfs/server/datanode/ReplicaInfo.java | 7 +
.../server/datanode/fsdataset/FsVolumeSpi.java | 11 +
.../datanode/fsdataset/impl/BlockPoolSlice.java | 6 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 15 +-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 58 +++-
.../server/datanode/TestDirectoryScanner.java | 8 +
.../fsdataset/impl/TestRbwSpaceReservation.java | 288 +++
.../fsdataset/impl/TestWriteToReplica.java | 2 +-
12 files changed, 423 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1fa5829/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 4412b30..3d43171 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -612,6 +612,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6862. Add missing timeout annotations to tests. (Xiaoyu Yao via
Arpit Agarwal)
+HDFS-6898. DN must reserve space for a full block when an RBW block is
+created. (Arpit Agarwal)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1fa5829/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
index 77fe543..240dcd0 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
@@ -48,7 +48,7 @@ public class HdfsConstants {
"org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol";
- public static final int MIN_BLOCKS_FOR_WRITE = 5;
+ public static final int MIN_BLOCKS_FOR_WRITE = 1;
// Long that indicates "leave current quota unchanged"
public static final long QUOTA_DONT_SET = Long.MAX_VALUE;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1fa5829/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java
index 728dd38..4a89493 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java
@@ -34,10 +34,12 @@ public class ReplicaBeingWritten extends ReplicaInPipeline {
* @param genStamp replica generation stamp
* @param vol volume where replica is located
* @param dir directory path where block and meta files are located
+ * @param bytesToReserve disk space to reserve for this replica, based on
+ * the estimated maximum block length.
*/
public ReplicaBeingWritten(long blockId, long genStamp,
-FsVolumeSpi vol, File dir) {
-super( blockId, genStamp, vol, dir);
+FsVolumeSpi vol, File dir, long bytesToReserve) {
+super(blockId, genStamp, vol, dir, bytesToReserve);
}
/**
@@ -60,10 +62,12 @@ public class ReplicaBeingWritten extends ReplicaInPipeline {
* @param vol volume where replica is located
* @param dir directory path where block and meta files are located
* @param writer a thread that is writing to this replica
+ *
[4/4] git commit: Merge trunk into HDFS-6581
Merge trunk into HDFS-6581
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb8284d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb8284d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb8284d5
Branch: refs/heads/HDFS-6581
Commit: eb8284d50e1aa9f196556ed20b4b5e3f330e65fe
Parents: 31bbeaf a23144f
Author: arp
Authored: Sun Sep 7 14:46:46 2014 -0700
Committer: arp
Committed: Sun Sep 7 14:46:46 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +
.../hadoop/hdfs/protocol/HdfsConstants.java | 2 +-
.../server/datanode/ReplicaBeingWritten.java| 12 +-
.../hdfs/server/datanode/ReplicaInPipeline.java | 33 ++-
.../hdfs/server/datanode/ReplicaInfo.java | 7 +
.../server/datanode/fsdataset/FsVolumeSpi.java | 13 +-
.../datanode/fsdataset/impl/BlockPoolSlice.java | 6 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 15 +-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 58 +++-
.../apache/hadoop/cli/TestCryptoAdminCLI.java | 4 +-
.../org/apache/hadoop/hdfs/TestDFSUtil.java | 4 +-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 12 +-
.../hadoop/hdfs/TestReservedRawPaths.java | 3 +-
.../server/datanode/TestDirectoryScanner.java | 8 +
.../fsdataset/impl/TestRbwSpaceReservation.java | 288 +++
.../fsdataset/impl/TestWriteToReplica.java | 2 +-
hadoop-yarn-project/CHANGES.txt | 3 +
.../yarn/webapp/util/TestWebAppUtils.java | 4 +-
18 files changed, 448 insertions(+), 32 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb8284d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb8284d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb8284d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
--
diff --cc
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
index e141cd7,cba23c3..8ebf2b4
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
@@@ -46,6 -46,14 +46,17 @@@ public interface FsVolumeSpi
public StorageType getStorageType();
+ /** Returns true if the volume is NOT backed by persistent storage. */
+ public boolean isTransientStorage();
- }
++
+ /**
+* Reserve disk space for an RBW block so a writer does not run out of
+* space before the block is full.
+*/
+ public void reserveSpaceForRbw(long bytesToReserve);
+
+ /**
+* Release disk space previously reserved for RBW block.
+*/
+ public void releaseReservedSpace(long bytesToRelease);
-}
++}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb8284d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb8284d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
--
diff --cc
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 606b067,4511f21..b243f27
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@@ -920,30 -872,12 +920,30 @@@ class FsDatasetImpl implements FsDatase
" and thus cannot be created.");
}
// create a new block
-FsVolumeImpl v = volumes.getNextVolume(storageType, b.getNumBytes());
-// create a rbw file to hold block in the designated volume
+FsVolumeImpl v;
+while (true) {
+ try {
+if (allowLazyPersist) {
+
[3/4] git commit: HDFS Credential Provider related Unit Test Failure. Contributed by Xiaoyu Yao.
HDFS Credential Provider related Unit Test Failure. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a23144fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a23144fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a23144fd
Branch: refs/heads/HDFS-6581
Commit: a23144fd8a1e399e431f2f272388ec109df37ab1
Parents: d1fa582
Author: cnauroth
Authored: Sun Sep 7 08:39:20 2014 -0700
Committer: cnauroth
Committed: Sun Sep 7 08:39:20 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/cli/TestCryptoAdminCLI.java | 4 +++-
.../test/java/org/apache/hadoop/hdfs/TestDFSUtil.java | 4 +++-
.../org/apache/hadoop/hdfs/TestEncryptionZones.java | 12 +++-
.../org/apache/hadoop/hdfs/TestReservedRawPaths.java| 3 ++-
5 files changed, 18 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23144fd/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 3d43171..5a30d0a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -615,6 +615,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6898. DN must reserve space for a full block when an RBW block is
created. (Arpit Agarwal)
+HDFS-7025. HDFS Credential Provider related Unit Test Failure.
+(Xiaoyu Yao via cnauroth)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23144fd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
index 1c83829..adeabfe 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderFactory;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HDFSPolicyProvider;
@@ -64,8 +65,9 @@ public class TestCryptoAdminCLI extends CLITestHelperDFS {
tmpDir = new File(System.getProperty("test.build.data", "target"),
UUID.randomUUID().toString()).getAbsoluteFile();
+final Path jksPath = new Path(tmpDir.toString(), "test.jks");
conf.set(KeyProviderFactory.KEY_PROVIDER_PATH,
-JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks");
+JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri());
dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
dfsCluster.waitClusterUp();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23144fd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
index 5ffd3b5..046265f 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@@ -809,8 +810,9 @@ public class TestDFSUtil {
"target/test-dir"));
Configuration conf = new Configuration();
+final Path jksPath = new Path(testDir.toString(), "test.jks");
final String ourUrl =
-JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks";
+JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
File file = new File(testDi
[1/4] git commit: YARN-2519. Credential Provider related unit tests failed on Windows. Contributed by Xiaoyu Yao.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-6581 31bbeaf38 -> eb8284d50
YARN-2519. Credential Provider related unit tests failed on Windows.
Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cbea1b10
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cbea1b10
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cbea1b10
Branch: refs/heads/HDFS-6581
Commit: cbea1b10efd871d04c648af18449dc724685db74
Parents: 88209ce
Author: cnauroth
Authored: Sat Sep 6 20:05:07 2014 -0700
Committer: cnauroth
Committed: Sat Sep 6 20:05:07 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java | 4 +++-
2 files changed, 6 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cbea1b10/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 34a206a..beafc22 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -287,6 +287,9 @@ Release 2.6.0 - UNRELEASED
YARN-2431. NM restart: cgroup is not removed for reacquired containers
(jlowe)
+YARN-2519. Credential Provider related unit tests failed on Windows.
+(Xiaoyu Yao via cnauroth)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cbea1b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
index 18600fd..2bd91b4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertEquals;
import java.io.File;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.http.HttpServer2.Builder;
import org.apache.hadoop.security.alias.CredentialProvider;
@@ -74,8 +75,9 @@ public class TestWebAppUtils {
"target/test-dir"));
Configuration conf = new Configuration();
+final Path jksPath = new Path(testDir.toString(), "test.jks");
final String ourUrl =
-JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks";
+JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
File file = new File(testDir, "test.jks");
file.delete();
git commit: YARN-2512. Allowed pattern matching for origins in CrossOriginFilter. Contributed by Jonathan Eagles.
Repository: hadoop
Updated Branches:
refs/heads/trunk a23144fd8 -> a092cdf32
YARN-2512. Allowed pattern matching for origins in CrossOriginFilter.
Contributed by Jonathan Eagles.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a092cdf3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a092cdf3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a092cdf3
Branch: refs/heads/trunk
Commit: a092cdf32de4d752456286a9f4dda533d8a62bca
Parents: a23144f
Author: Zhijie Shen
Authored: Sun Sep 7 17:49:06 2014 -0700
Committer: Zhijie Shen
Committed: Sun Sep 7 17:49:06 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../timeline/webapp/CrossOriginFilter.java | 20 ++-
.../timeline/webapp/TestCrossOriginFilter.java | 21 +++-
3 files changed, 42 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a092cdf3/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index beafc22..ed31479 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -184,6 +184,9 @@ Release 2.6.0 - UNRELEASED
YARN-2508. Cross Origin configuration parameters prefix are not honored
(Mit Desai via jeagles)
+YARN-2512. Allowed pattern matching for origins in CrossOriginFilter.
+(Jonathan Eagles via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a092cdf3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
index d71175f..5a0703d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
@@ -24,6 +24,8 @@ import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import javax.servlet.Filter;
import javax.servlet.FilterChain;
@@ -204,7 +206,23 @@ public class CrossOriginFilter implements Filter {
@VisibleForTesting
boolean isOriginAllowed(String origin) {
-return allowAllOrigins || allowedOrigins.contains(origin);
+if (allowAllOrigins) {
+ return true;
+}
+
+for (String allowedOrigin : allowedOrigins) {
+ if (allowedOrigin.contains("*")) {
+String regex = allowedOrigin.replace(".", "\\.").replace("*", ".*");
+Pattern p = Pattern.compile(regex);
+Matcher m = p.matcher(origin);
+if (m.matches()) {
+ return true;
+}
+ } else if (allowedOrigin.equals(origin)) {
+return true;
+ }
+}
+return false;
}
private boolean areHeadersAllowed(String accessControlRequestHeaders) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a092cdf3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
index f666c21..ccc9bbf 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
@@ -77,7 +77,26 @@ public class TestCrossOriginFilter {
git commit: YARN-2512. Allowed pattern matching for origins in CrossOriginFilter. Contributed by Jonathan Eagles.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 b87d1d7d0 -> f2a5a5d4a
YARN-2512. Allowed pattern matching for origins in CrossOriginFilter.
Contributed by Jonathan Eagles.
(cherry picked from commit a092cdf32de4d752456286a9f4dda533d8a62bca)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f2a5a5d4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f2a5a5d4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f2a5a5d4
Branch: refs/heads/branch-2
Commit: f2a5a5d4aada5b0a3660e5db281f505e2cd3193a
Parents: b87d1d7
Author: Zhijie Shen
Authored: Sun Sep 7 17:49:06 2014 -0700
Committer: Zhijie Shen
Committed: Sun Sep 7 17:52:38 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../timeline/webapp/CrossOriginFilter.java | 20 ++-
.../timeline/webapp/TestCrossOriginFilter.java | 21 +++-
3 files changed, 42 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2a5a5d4/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 83274b6..a149028 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -161,6 +161,9 @@ Release 2.6.0 - UNRELEASED
YARN-2508. Cross Origin configuration parameters prefix are not honored
(Mit Desai via jeagles)
+YARN-2512. Allowed pattern matching for origins in CrossOriginFilter.
+(Jonathan Eagles via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2a5a5d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
index d71175f..5a0703d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
@@ -24,6 +24,8 @@ import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import javax.servlet.Filter;
import javax.servlet.FilterChain;
@@ -204,7 +206,23 @@ public class CrossOriginFilter implements Filter {
@VisibleForTesting
boolean isOriginAllowed(String origin) {
-return allowAllOrigins || allowedOrigins.contains(origin);
+if (allowAllOrigins) {
+ return true;
+}
+
+for (String allowedOrigin : allowedOrigins) {
+ if (allowedOrigin.contains("*")) {
+String regex = allowedOrigin.replace(".", "\\.").replace("*", ".*");
+Pattern p = Pattern.compile(regex);
+Matcher m = p.matcher(origin);
+if (m.matches()) {
+ return true;
+}
+ } else if (allowedOrigin.equals(origin)) {
+return true;
+ }
+}
+return false;
}
private boolean areHeadersAllowed(String accessControlRequestHeaders) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2a5a5d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
index f666c21..ccc9bbf 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossO
git commit: YARN-2507. Documented CrossOriginFilter configurations for the timeline server. Contributed by Jonathan Eagles.
Repository: hadoop Updated Branches: refs/heads/trunk a092cdf32 -> 56dc496a1 YARN-2507. Documented CrossOriginFilter configurations for the timeline server. Contributed by Jonathan Eagles. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/56dc496a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/56dc496a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/56dc496a Branch: refs/heads/trunk Commit: 56dc496a1031621d2b701801de4ec29179d75f2e Parents: a092cdf Author: Zhijie Shen Authored: Sun Sep 7 18:22:40 2014 -0700 Committer: Zhijie Shen Committed: Sun Sep 7 18:22:40 2014 -0700 -- hadoop-yarn-project/CHANGES.txt | 3 ++ .../src/site/apt/TimelineServer.apt.vm | 37 2 files changed, 40 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/56dc496a/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index ed31479..ed9de87 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -187,6 +187,9 @@ Release 2.6.0 - UNRELEASED YARN-2512. Allowed pattern matching for origins in CrossOriginFilter. (Jonathan Eagles via zjshen) +YARN-2507. Documented CrossOriginFilter configurations for the timeline +server. (Jonathan Eagles via zjshen) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/56dc496a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm index c704d37..92c7377 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm @@ -102,6 +102,43 @@ YARN Timeline Server yarn.timeline-service.handler-thread-count 10 + + + Enables cross-origin support (CORS) for web services where + cross-origin web response headers are needed. For example, javascript making + a web services request to the timeline server. + yarn.timeline-service.http-cross-origin.enabled + false + + + + Comma separated list of origins that are allowed for web + services needing cross-origin (CORS) support. Wildcards (*) and patterns + allowed + yarn.timeline-service.http-cross-origin.allowed-origins + * + + + + Comma separated list of methods that are allowed for web + services needing cross-origin (CORS) support. + yarn.timeline-service.http-cross-origin.allowed-methods + GET,POST,HEAD + + + + Comma separated list of headers that are allowed for web + services needing cross-origin (CORS) support. + yarn.timeline-service.http-cross-origin.allowed-headers + X-Requested-With,Content-Type,Accept,Origin + + + + The number of seconds a pre-flighted request can be cached + for web services needing cross-origin (CORS) support. + yarn.timeline-service.http-cross-origin.max-age + 1800 + +---+ * Generic-data related Configuration
git commit: YARN-2507. Documented CrossOriginFilter configurations for the timeline server. Contributed by Jonathan Eagles.
Repository: hadoop Updated Branches: refs/heads/branch-2 f2a5a5d4a -> 0974400b7 YARN-2507. Documented CrossOriginFilter configurations for the timeline server. Contributed by Jonathan Eagles. (cherry picked from commit 56dc496a1031621d2b701801de4ec29179d75f2e) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0974400b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0974400b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0974400b Branch: refs/heads/branch-2 Commit: 0974400b7401cebef0e9212d7400d868271876f9 Parents: f2a5a5d Author: Zhijie Shen Authored: Sun Sep 7 18:22:40 2014 -0700 Committer: Zhijie Shen Committed: Sun Sep 7 18:25:13 2014 -0700 -- hadoop-yarn-project/CHANGES.txt | 3 ++ .../src/site/apt/TimelineServer.apt.vm | 37 2 files changed, 40 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0974400b/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index a149028..8b362a5 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -164,6 +164,9 @@ Release 2.6.0 - UNRELEASED YARN-2512. Allowed pattern matching for origins in CrossOriginFilter. (Jonathan Eagles via zjshen) +YARN-2507. Documented CrossOriginFilter configurations for the timeline +server. (Jonathan Eagles via zjshen) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/0974400b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm index c704d37..92c7377 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm @@ -102,6 +102,43 @@ YARN Timeline Server yarn.timeline-service.handler-thread-count 10 + + + Enables cross-origin support (CORS) for web services where + cross-origin web response headers are needed. For example, javascript making + a web services request to the timeline server. + yarn.timeline-service.http-cross-origin.enabled + false + + + + Comma separated list of origins that are allowed for web + services needing cross-origin (CORS) support. Wildcards (*) and patterns + allowed + yarn.timeline-service.http-cross-origin.allowed-origins + * + + + + Comma separated list of methods that are allowed for web + services needing cross-origin (CORS) support. + yarn.timeline-service.http-cross-origin.allowed-methods + GET,POST,HEAD + + + + Comma separated list of headers that are allowed for web + services needing cross-origin (CORS) support. + yarn.timeline-service.http-cross-origin.allowed-headers + X-Requested-With,Content-Type,Accept,Origin + + + + The number of seconds a pre-flighted request can be cached + for web services needing cross-origin (CORS) support. + yarn.timeline-service.http-cross-origin.max-age + 1800 + +---+ * Generic-data related Configuration
git commit: YARN-2515. Updated ConverterUtils#toContainerId to parse epoch. Contributed by Tsuyoshi OZAWA
Repository: hadoop
Updated Branches:
refs/heads/trunk 56dc496a1 -> 0974f434c
YARN-2515. Updated ConverterUtils#toContainerId to parse epoch. Contributed by
Tsuyoshi OZAWA
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0974f434
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0974f434
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0974f434
Branch: refs/heads/trunk
Commit: 0974f434c47ffbf4b77a8478937fd99106c8ddbd
Parents: 56dc496
Author: Jian He
Authored: Sun Sep 7 18:25:44 2014 -0700
Committer: Jian He
Committed: Sun Sep 7 18:29:22 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../hadoop/yarn/api/records/ContainerId.java| 39 +++-
.../apache/hadoop/yarn/util/ConverterUtils.java | 15 +---
.../apache/hadoop/yarn/api/TestContainerId.java | 4 ++
.../hadoop/yarn/util/TestConverterUtils.java| 9 +
5 files changed, 55 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0974f434/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index ed9de87..d54fcd6 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -190,6 +190,9 @@ Release 2.6.0 - UNRELEASED
YARN-2507. Documented CrossOriginFilter configurations for the timeline
server. (Jonathan Eagles via zjshen)
+YARN-2515. Updated ConverterUtils#toContainerId to parse epoch.
+(Tsuyoshi OZAWA via jianhe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0974f434/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java
index fc7f404..321052b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java
@@ -18,8 +18,10 @@
package org.apache.hadoop.yarn.api.records;
-import java.text.NumberFormat;
+import com.google.common.base.Splitter;
+import java.text.NumberFormat;
+import java.util.Iterator;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Stable;
@@ -33,6 +35,8 @@ import org.apache.hadoop.yarn.util.Records;
@Public
@Stable
public abstract class ContainerId implements Comparable{
+ private static final Splitter _SPLITTER = Splitter.on('_').trimResults();
+ private static final String CONTAINER_PREFIX = "container";
@Private
@Unstable
@@ -163,5 +167,38 @@ public abstract class ContainerId implements
Comparable{
return sb.toString();
}
+ @Public
+ @Unstable
+ public static ContainerId fromString(String containerIdStr) {
+Iterator it = _SPLITTER.split(containerIdStr).iterator();
+if (!it.next().equals(CONTAINER_PREFIX)) {
+ throw new IllegalArgumentException("Invalid ContainerId prefix: "
+ + containerIdStr);
+}
+try {
+ ApplicationAttemptId appAttemptID = toApplicationAttemptId(it);
+ int id = Integer.parseInt(it.next());
+ int epoch = 0;
+ if (it.hasNext()) {
+epoch = Integer.parseInt(it.next());
+ }
+ int cid = (epoch << 22) | id;
+ ContainerId containerId = ContainerId.newInstance(appAttemptID, cid);
+ return containerId;
+} catch (NumberFormatException n) {
+ throw new IllegalArgumentException("Invalid ContainerId: "
+ + containerIdStr, n);
+}
+ }
+
+ private static ApplicationAttemptId toApplicationAttemptId(
+ Iterator it) throws NumberFormatException {
+ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
+Integer.parseInt(it.next()));
+ApplicationAttemptId appAttemptId =
+ApplicationAttemptId.newInstance(appId, Integer.parseInt(it.next()));
+return appAttemptId;
+ }
+
protected abstract void build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0974f434/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hado
git commit: YARN-2515. Updated ConverterUtils#toContainerId to parse epoch. Contributed by Tsuyoshi OZAWA (cherry picked from commit 0974f434c47ffbf4b77a8478937fd99106c8ddbd)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 0974400b7 -> c17810a84
YARN-2515. Updated ConverterUtils#toContainerId to parse epoch. Contributed by
Tsuyoshi OZAWA
(cherry picked from commit 0974f434c47ffbf4b77a8478937fd99106c8ddbd)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c17810a8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c17810a8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c17810a8
Branch: refs/heads/branch-2
Commit: c17810a84c98c808403fe4caf4532efa1e781221
Parents: 0974400
Author: Jian He
Authored: Sun Sep 7 18:25:44 2014 -0700
Committer: Jian He
Committed: Sun Sep 7 18:31:07 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../hadoop/yarn/api/records/ContainerId.java| 39 +++-
.../apache/hadoop/yarn/util/ConverterUtils.java | 15 +---
.../apache/hadoop/yarn/api/TestContainerId.java | 4 ++
.../hadoop/yarn/util/TestConverterUtils.java| 9 +
5 files changed, 55 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c17810a8/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 8b362a5..9124678 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -167,6 +167,9 @@ Release 2.6.0 - UNRELEASED
YARN-2507. Documented CrossOriginFilter configurations for the timeline
server. (Jonathan Eagles via zjshen)
+YARN-2515. Updated ConverterUtils#toContainerId to parse epoch.
+(Tsuyoshi OZAWA via jianhe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c17810a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java
index fc7f404..321052b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java
@@ -18,8 +18,10 @@
package org.apache.hadoop.yarn.api.records;
-import java.text.NumberFormat;
+import com.google.common.base.Splitter;
+import java.text.NumberFormat;
+import java.util.Iterator;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Stable;
@@ -33,6 +35,8 @@ import org.apache.hadoop.yarn.util.Records;
@Public
@Stable
public abstract class ContainerId implements Comparable{
+ private static final Splitter _SPLITTER = Splitter.on('_').trimResults();
+ private static final String CONTAINER_PREFIX = "container";
@Private
@Unstable
@@ -163,5 +167,38 @@ public abstract class ContainerId implements
Comparable{
return sb.toString();
}
+ @Public
+ @Unstable
+ public static ContainerId fromString(String containerIdStr) {
+Iterator it = _SPLITTER.split(containerIdStr).iterator();
+if (!it.next().equals(CONTAINER_PREFIX)) {
+ throw new IllegalArgumentException("Invalid ContainerId prefix: "
+ + containerIdStr);
+}
+try {
+ ApplicationAttemptId appAttemptID = toApplicationAttemptId(it);
+ int id = Integer.parseInt(it.next());
+ int epoch = 0;
+ if (it.hasNext()) {
+epoch = Integer.parseInt(it.next());
+ }
+ int cid = (epoch << 22) | id;
+ ContainerId containerId = ContainerId.newInstance(appAttemptID, cid);
+ return containerId;
+} catch (NumberFormatException n) {
+ throw new IllegalArgumentException("Invalid ContainerId: "
+ + containerIdStr, n);
+}
+ }
+
+ private static ApplicationAttemptId toApplicationAttemptId(
+ Iterator it) throws NumberFormatException {
+ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
+Integer.parseInt(it.next()));
+ApplicationAttemptId appAttemptId =
+ApplicationAttemptId.newInstance(appId, Integer.parseInt(it.next()));
+return appAttemptId;
+ }
+
protected abstract void build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c17810a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
--
diff --git
a/hadoo
[03/19] git commit: HDFS-6862. Add missing timeout annotations to tests. (Contributed by Xiaoyu Yao)
HDFS-6862. Add missing timeout annotations to tests. (Contributed by Xiaoyu Yao)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9609b730
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9609b730
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9609b730
Branch: refs/heads/HDFS-6584
Commit: 9609b7303a98c8eff676c5a086b08b1ca9ab777c
Parents: b051327
Author: arp
Authored: Fri Sep 5 11:08:03 2014 -0700
Committer: arp
Committed: Fri Sep 5 11:10:58 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../apache/hadoop/hdfs/TestHDFSServerPorts.java | 24
.../TestValidateConfigurationSettings.java | 16 ++---
.../namenode/ha/TestDelegationTokensWithHA.java | 63
.../hdfs/server/namenode/ha/TestHAMetrics.java | 10 ++--
.../namenode/ha/TestHAStateTransitions.java | 52 +++-
.../namenode/ha/TestStandbyCheckpoints.java | 54 +++--
7 files changed, 97 insertions(+), 125 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9609b730/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 7b8917b..0772ea6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -607,6 +607,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6979. hdfs.dll does not produce .pdb files. (cnauroth)
+HDFS-6862. Add missing timeout annotations to tests. (Xiaoyu Yao via
+Arpit Agarwal)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9609b730/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSServerPorts.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSServerPorts.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSServerPorts.java
index 59d1615..ce8a4e7 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSServerPorts.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSServerPorts.java
@@ -17,14 +17,6 @@
*/
package org.apache.hadoop.hdfs;
-import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.UnknownHostException;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -39,6 +31,14 @@ import org.apache.hadoop.net.DNS;
import org.apache.hadoop.test.PathUtils;
import org.junit.Test;
+import java.io.File;
+import java.io.IOException;
+import java.net.UnknownHostException;
+
+import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
/**
* This test checks correctness of port usage by hdfs components:
* NameNode, DataNode, SecondaryNamenode and BackupNode.
@@ -245,7 +245,7 @@ public class TestHDFSServerPorts {
return true;
}
- @Test
+ @Test(timeout = 30)
public void testNameNodePorts() throws Exception {
runTestNameNodePorts(false);
runTestNameNodePorts(true);
@@ -296,7 +296,7 @@ public class TestHDFSServerPorts {
/**
* Verify datanode port usage.
*/
- @Test
+ @Test(timeout = 30)
public void testDataNodePorts() throws Exception {
NameNode nn = null;
try {
@@ -332,7 +332,7 @@ public class TestHDFSServerPorts {
/**
* Verify secondary namenode port usage.
*/
- @Test
+ @Test(timeout = 30)
public void testSecondaryNodePorts() throws Exception {
NameNode nn = null;
try {
@@ -361,7 +361,7 @@ public class TestHDFSServerPorts {
/**
* Verify BackupNode port usage.
*/
- @Test
+@Test(timeout = 30)
public void testBackupNodePorts() throws Exception {
NameNode nn = null;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9609b730/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestValidateConfigurationSettings.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestValidateConfigurationSettings.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs
[18/19] git commit: YARN-2515. Updated ConverterUtils#toContainerId to parse epoch. Contributed by Tsuyoshi OZAWA
YARN-2515. Updated ConverterUtils#toContainerId to parse epoch. Contributed by
Tsuyoshi OZAWA
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0974f434
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0974f434
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0974f434
Branch: refs/heads/HDFS-6584
Commit: 0974f434c47ffbf4b77a8478937fd99106c8ddbd
Parents: 56dc496
Author: Jian He
Authored: Sun Sep 7 18:25:44 2014 -0700
Committer: Jian He
Committed: Sun Sep 7 18:29:22 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../hadoop/yarn/api/records/ContainerId.java| 39 +++-
.../apache/hadoop/yarn/util/ConverterUtils.java | 15 +---
.../apache/hadoop/yarn/api/TestContainerId.java | 4 ++
.../hadoop/yarn/util/TestConverterUtils.java| 9 +
5 files changed, 55 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0974f434/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index ed9de87..d54fcd6 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -190,6 +190,9 @@ Release 2.6.0 - UNRELEASED
YARN-2507. Documented CrossOriginFilter configurations for the timeline
server. (Jonathan Eagles via zjshen)
+YARN-2515. Updated ConverterUtils#toContainerId to parse epoch.
+(Tsuyoshi OZAWA via jianhe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0974f434/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java
index fc7f404..321052b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java
@@ -18,8 +18,10 @@
package org.apache.hadoop.yarn.api.records;
-import java.text.NumberFormat;
+import com.google.common.base.Splitter;
+import java.text.NumberFormat;
+import java.util.Iterator;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Stable;
@@ -33,6 +35,8 @@ import org.apache.hadoop.yarn.util.Records;
@Public
@Stable
public abstract class ContainerId implements Comparable{
+ private static final Splitter _SPLITTER = Splitter.on('_').trimResults();
+ private static final String CONTAINER_PREFIX = "container";
@Private
@Unstable
@@ -163,5 +167,38 @@ public abstract class ContainerId implements
Comparable{
return sb.toString();
}
+ @Public
+ @Unstable
+ public static ContainerId fromString(String containerIdStr) {
+Iterator it = _SPLITTER.split(containerIdStr).iterator();
+if (!it.next().equals(CONTAINER_PREFIX)) {
+ throw new IllegalArgumentException("Invalid ContainerId prefix: "
+ + containerIdStr);
+}
+try {
+ ApplicationAttemptId appAttemptID = toApplicationAttemptId(it);
+ int id = Integer.parseInt(it.next());
+ int epoch = 0;
+ if (it.hasNext()) {
+epoch = Integer.parseInt(it.next());
+ }
+ int cid = (epoch << 22) | id;
+ ContainerId containerId = ContainerId.newInstance(appAttemptID, cid);
+ return containerId;
+} catch (NumberFormatException n) {
+ throw new IllegalArgumentException("Invalid ContainerId: "
+ + containerIdStr, n);
+}
+ }
+
+ private static ApplicationAttemptId toApplicationAttemptId(
+ Iterator it) throws NumberFormatException {
+ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
+Integer.parseInt(it.next()));
+ApplicationAttemptId appAttemptId =
+ApplicationAttemptId.newInstance(appId, Integer.parseInt(it.next()));
+return appAttemptId;
+ }
+
protected abstract void build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0974f434/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-y
[02/19] git commit: HDFS-6979. Fix minor error in CHANGES.txt. Contributed by Chris Nauroth.
HDFS-6979. Fix minor error in CHANGES.txt. Contributed by Chris Nauroth. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b051327a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b051327a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b051327a Branch: refs/heads/HDFS-6584 Commit: b051327ab6a01774e1dad59e1e547dd16f603789 Parents: fab9bc5 Author: cnauroth Authored: Fri Sep 5 11:07:41 2014 -0700 Committer: cnauroth Committed: Fri Sep 5 11:07:41 2014 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b051327a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 680af55..7b8917b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -605,7 +605,7 @@ Release 2.6.0 - UNRELEASED HDFS-6831. Inconsistency between 'hdfs dfsadmin' and 'hdfs dfsadmin -help'. (Xiaoyu Yao via Arpit Agarwal) -HDFS-6979. hdfs.dll not produce .pdb files. (cnauroth) +HDFS-6979. hdfs.dll does not produce .pdb files. (cnauroth) BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
[06/19] git commit: HADOOP-11065. Rat check should exclude **/build/**. (kasha)
HADOOP-11065. Rat check should exclude **/build/**. (kasha) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0571b456 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0571b456 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0571b456 Branch: refs/heads/HDFS-6584 Commit: 0571b4561bad7e0230920e52d3758a3658fcf20d Parents: 7a62515 Author: Karthik Kambatla Authored: Fri Sep 5 13:08:59 2014 -0700 Committer: Karthik Kambatla Committed: Fri Sep 5 13:09:10 2014 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ pom.xml | 1 + 2 files changed, 3 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0571b456/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index afd1cc7..d20bf08 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -777,6 +777,8 @@ Release 2.5.1 - UNRELEASED HADOOP-11001. Fix test-patch to work with the git repo. (kasha) +HADOOP-11065. Rat check should exclude "**/build/**". (kasha) + Release 2.5.0 - 2014-08-11 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/0571b456/pom.xml -- diff --git a/pom.xml b/pom.xml index a4f8241..5cc30c2 100644 --- a/pom.xml +++ b/pom.xml @@ -324,6 +324,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs .gitignore .git/** .idea/** + **/build/**
[11/19] git commit: HDFS-6986. DistributedFileSystem must get delegation tokens from configured KeyProvider. (zhz via tucu)
HDFS-6986. DistributedFileSystem must get delegation tokens from configured
KeyProvider. (zhz via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3b35f816
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3b35f816
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3b35f816
Branch: refs/heads/HDFS-6584
Commit: 3b35f81603bbfae119762b50bcb46de70a421368
Parents: 0f3c19c
Author: Alejandro Abdelnur
Authored: Fri Sep 5 22:33:48 2014 -0700
Committer: Alejandro Abdelnur
Committed: Fri Sep 5 22:33:48 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../java/org/apache/hadoop/hdfs/DFSClient.java | 4 ++
.../hadoop/hdfs/DistributedFileSystem.java | 24 +++
.../apache/hadoop/hdfs/TestEncryptionZones.java | 43
4 files changed, 74 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b35f816/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0772ea6..333bdce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -711,6 +711,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6714. TestBlocksScheduledCounter#testBlocksScheduledCounter should
shutdown cluster (vinayakumarb)
+ HDFS-6986. DistributedFileSystem must get delegation tokens from
configured
+ KeyProvider. (zhz via tucu)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b35f816/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 8daf912..e4215f0 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -3084,4 +3084,8 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
DFSHedgedReadMetrics getHedgedReadMetrics() {
return HEDGED_READ_METRIC;
}
+
+ public KeyProviderCryptoExtension getKeyProvider() {
+return provider;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b35f816/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index bf7d62e..dbdf5c1 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -84,8 +84,10 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@@ -1946,6 +1948,28 @@ public class DistributedFileSystem extends FileSystem {
}.resolve(this, absF);
}
+ @Override
+ public Token[] addDelegationTokens(
+ final String renewer, Credentials credentials) throws IOException {
+Token[] tokens = super.addDelegationTokens(renewer, credentials);
+if (dfs.getKeyProvider() != null) {
+ KeyProviderDelegationTokenExtension keyProviderDelegationTokenExtension =
+ KeyProviderDelegationTokenExtension.
+ createKeyProviderDelegationTokenExtension(dfs.getKeyProvider());
+ Token[] kpTokens = keyProviderDelegationTokenExtension.
+ addDelegationTokens(renewer, credentials);
+ if (tokens != null && kpTokens != null) {
+Token[] all = new Token[tokens.length + kpTokens.length];
+System.arraycopy(tokens, 0, all, 0, tokens.length);
+System.arraycopy(kpTokens, 0, all, tokens.length, kpTokens.length);
+tokens = all;
+ } else {
+tokens = (tokens != null) ? tokens : kpTokens;
+ }
+}
+return tokens;
+ }
+
public DFSInotifyEventInputStream getInotifyEve
[13/19] git commit: YARN-2519. Credential Provider related unit tests failed on Windows. Contributed by Xiaoyu Yao.
YARN-2519. Credential Provider related unit tests failed on Windows.
Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cbea1b10
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cbea1b10
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cbea1b10
Branch: refs/heads/HDFS-6584
Commit: cbea1b10efd871d04c648af18449dc724685db74
Parents: 88209ce
Author: cnauroth
Authored: Sat Sep 6 20:05:07 2014 -0700
Committer: cnauroth
Committed: Sat Sep 6 20:05:07 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java | 4 +++-
2 files changed, 6 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cbea1b10/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 34a206a..beafc22 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -287,6 +287,9 @@ Release 2.6.0 - UNRELEASED
YARN-2431. NM restart: cgroup is not removed for reacquired containers
(jlowe)
+YARN-2519. Credential Provider related unit tests failed on Windows.
+(Xiaoyu Yao via cnauroth)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cbea1b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
index 18600fd..2bd91b4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertEquals;
import java.io.File;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.http.HttpServer2.Builder;
import org.apache.hadoop.security.alias.CredentialProvider;
@@ -74,8 +75,9 @@ public class TestWebAppUtils {
"target/test-dir"));
Configuration conf = new Configuration();
+final Path jksPath = new Path(testDir.toString(), "test.jks");
final String ourUrl =
-JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks";
+JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
File file = new File(testDir, "test.jks");
file.delete();
[19/19] git commit: Merge branch 'trunk' into HDFS-6584
Merge branch 'trunk' into HDFS-6584
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f1432e24
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f1432e24
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f1432e24
Branch: refs/heads/HDFS-6584
Commit: f1432e24244034f2009fc91e4bae390c6fcc78b0
Parents: 22a41dc 0974f43
Author: Tsz-Wo Nicholas Sze
Authored: Mon Sep 8 10:54:48 2014 +0800
Committer: Tsz-Wo Nicholas Sze
Committed: Mon Sep 8 10:54:48 2014 +0800
--
hadoop-common-project/hadoop-common/CHANGES.txt | 8 +
.../crypto/key/kms/KMSClientProvider.java | 6 +-
hadoop-common-project/hadoop-kms/pom.xml| 4 +-
.../hadoop/crypto/key/kms/server/MiniKMS.java | 197 +
.../hadoop/crypto/key/kms/server/TestKMS.java | 88 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 12 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 4 +
.../hadoop/hdfs/DistributedFileSystem.java | 24 ++
.../hadoop/hdfs/protocol/HdfsConstants.java | 2 +-
.../server/blockmanagement/BlockManager.java| 21 +-
.../server/blockmanagement/DatanodeManager.java | 6 +-
.../server/blockmanagement/HostFileManager.java | 4 +
.../server/datanode/ReplicaBeingWritten.java| 12 +-
.../hdfs/server/datanode/ReplicaInPipeline.java | 33 ++-
.../hdfs/server/datanode/ReplicaInfo.java | 7 +
.../server/datanode/fsdataset/FsVolumeSpi.java | 11 +
.../datanode/fsdataset/impl/BlockPoolSlice.java | 6 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 15 +-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 58 +++-
.../hdfs/server/namenode/FSNamesystem.java | 46 +--
.../apache/hadoop/cli/TestCryptoAdminCLI.java | 4 +-
.../org/apache/hadoop/hdfs/TestDFSUtil.java | 4 +-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 55 +++-
.../hadoop/hdfs/TestReservedRawPaths.java | 3 +-
.../server/datanode/TestDirectoryScanner.java | 8 +
.../fsdataset/impl/TestRbwSpaceReservation.java | 288 +++
.../fsdataset/impl/TestWriteToReplica.java | 2 +-
.../hdfs/server/namenode/NameNodeAdapter.java | 2 +-
hadoop-yarn-project/CHANGES.txt | 15 +
.../hadoop/yarn/api/records/ContainerId.java| 39 ++-
.../apache/hadoop/yarn/util/ConverterUtils.java | 15 +-
.../apache/hadoop/yarn/api/TestContainerId.java | 4 +
.../hadoop/yarn/util/TestConverterUtils.java| 9 +
.../yarn/webapp/util/TestWebAppUtils.java | 4 +-
.../timeline/webapp/CrossOriginFilter.java | 20 +-
.../webapp/CrossOriginFilterInitializer.java| 12 +-
.../timeline/webapp/TestCrossOriginFilter.java | 21 +-
.../TestCrossOriginFilterInitializer.java | 7 +-
.../src/site/apt/TimelineServer.apt.vm | 37 +++
39 files changed, 952 insertions(+), 161 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1432e24/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1432e24/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1432e24/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1432e24/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1432e24/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
--
diff --cc
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 956900d,6176188..9b030e7
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@@ -398,10 -396,22 +399,26 @@@ public class BlockManager
}
}
+ public BlockStoragePolicy getStoragePolicy(final String policyName) {
+return storagePolicySuite.getPolicy(policyName);
+ }
+
+ public long getReplicationRecheckInterval() {
+ return replicationRecheckInterv
[08/19] git commit: YARN-2508. Cross Origin configuration parameters prefix are not honored (Mit Desai via jeagles)
YARN-2508. Cross Origin configuration parameters prefix are not honored (Mit
Desai via jeagles)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e6420fec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e6420fec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e6420fec
Branch: refs/heads/HDFS-6584
Commit: e6420fec0af9b8d4f424098688ae4926ff527fcf
Parents: 21c0cde
Author: Jonathan Eagles
Authored: Fri Sep 5 19:42:40 2014 -0500
Committer: Jonathan Eagles
Committed: Fri Sep 5 19:42:40 2014 -0500
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../timeline/webapp/CrossOriginFilterInitializer.java | 12 +++-
.../webapp/TestCrossOriginFilterInitializer.java| 7 ++-
3 files changed, 16 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6420fec/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 1a5ea07..34a206a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -181,6 +181,9 @@ Release 2.6.0 - UNRELEASED
YARN-2511. Allowed all origins by default when CrossOriginFilter is
enabled. (Jonathan Eagles via zjshen)
+YARN-2508. Cross Origin configuration parameters prefix are not honored
+(Mit Desai via jeagles)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6420fec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilterInitializer.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilterInitializer.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilterInitializer.java
index 69e0188..148cc63 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilterInitializer.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilterInitializer.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.timeline.webapp;
+import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
@@ -37,6 +38,15 @@ public class CrossOriginFilterInitializer extends
FilterInitializer {
}
static Map getFilterParameters(Configuration conf) {
-return conf.getValByRegex(PREFIX);
+Map filterParams =
+new HashMap();
+for (Map.Entry entry : conf.getValByRegex(PREFIX)
+.entrySet()) {
+ String name = entry.getKey();
+ String value = entry.getValue();
+ name = name.substring(PREFIX.length());
+ filterParams.put(name, value);
+}
+return filterParams;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6420fec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilterInitializer.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilterInitializer.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilterInitializer.java
index 3199aac..cf26368 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilterInitializer.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilterInitializer.java
@@ -42,11 +42,8 @@ public class TestCrossOriginFilterInitializer {
CrossOriginFilterInitializer.getFilterParameters(conf);
// retrieve values
-String rootvalue =
-filterParameters.get(CrossOriginFilterInitializer.PREFIX +
"rootparam");
-String nestedvalue =
-filterParameters.get(CrossOriginFilterIni
[15/19] git commit: HDFS Credential Provider related Unit Test Failure. Contributed by Xiaoyu Yao.
HDFS Credential Provider related Unit Test Failure. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a23144fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a23144fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a23144fd
Branch: refs/heads/HDFS-6584
Commit: a23144fd8a1e399e431f2f272388ec109df37ab1
Parents: d1fa582
Author: cnauroth
Authored: Sun Sep 7 08:39:20 2014 -0700
Committer: cnauroth
Committed: Sun Sep 7 08:39:20 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/cli/TestCryptoAdminCLI.java | 4 +++-
.../test/java/org/apache/hadoop/hdfs/TestDFSUtil.java | 4 +++-
.../org/apache/hadoop/hdfs/TestEncryptionZones.java | 12 +++-
.../org/apache/hadoop/hdfs/TestReservedRawPaths.java| 3 ++-
5 files changed, 18 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23144fd/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 3d43171..5a30d0a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -615,6 +615,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6898. DN must reserve space for a full block when an RBW block is
created. (Arpit Agarwal)
+HDFS-7025. HDFS Credential Provider related Unit Test Failure.
+(Xiaoyu Yao via cnauroth)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23144fd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
index 1c83829..adeabfe 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderFactory;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HDFSPolicyProvider;
@@ -64,8 +65,9 @@ public class TestCryptoAdminCLI extends CLITestHelperDFS {
tmpDir = new File(System.getProperty("test.build.data", "target"),
UUID.randomUUID().toString()).getAbsoluteFile();
+final Path jksPath = new Path(tmpDir.toString(), "test.jks");
conf.set(KeyProviderFactory.KEY_PROVIDER_PATH,
-JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks");
+JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri());
dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
dfsCluster.waitClusterUp();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23144fd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
index 5ffd3b5..046265f 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@@ -809,8 +810,9 @@ public class TestDFSUtil {
"target/test-dir"));
Configuration conf = new Configuration();
+final Path jksPath = new Path(testDir.toString(), "test.jks");
final String ourUrl =
-JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks";
+JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
File file = new File(testDi
[04/19] git commit: HDFS-6998. warning message 'ssl.client.truststore.location has not been set' gets printed for hftp command. (Contributed by Xiaoyu Yao)
HDFS-6998. warning message 'ssl.client.truststore.location has not been set'
gets printed for hftp command. (Contributed by Xiaoyu Yao)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71269f70
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71269f70
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71269f70
Branch: refs/heads/HDFS-6584
Commit: 71269f70971dc7aa7bcb5e78b19cb3f04fdaa2f4
Parents: 9609b73
Author: arp
Authored: Fri Sep 5 11:14:10 2014 -0700
Committer: arp
Committed: Fri Sep 5 11:14:10 2014 -0700
--
.../org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
2 files changed, 4 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71269f70/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java
index aabb815..4b81e17 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java
@@ -212,7 +212,7 @@ public class FileBasedKeyStoresFactory implements
KeyStoresFactory {
LOG.debug(mode.toString() + " Loaded TrustStore: " + truststoreLocation);
trustManagers = new TrustManager[]{trustManager};
} else {
- LOG.warn("The property '" + locationProperty + "' has not been set, " +
+ LOG.debug("The property '" + locationProperty + "' has not been set, " +
"no TrustStore will be loaded");
trustManagers = null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71269f70/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0772ea6..5c4aeea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -610,6 +610,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6862. Add missing timeout annotations to tests. (Xiaoyu Yao via
Arpit Agarwal)
+HDFS-6998. warning message 'ssl.client.truststore.location has not been
+set' gets printed for hftp command. (Xiaoyu Yao via Arpit Agarwal)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
[05/19] git commit: HADOOP-11052. hadoop_verify_secure_prereq's results aren't checked in bin/hdfs (aw)
HADOOP-11052. hadoop_verify_secure_prereq's results aren't checked in bin/hdfs
(aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7a62515c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7a62515c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7a62515c
Branch: refs/heads/HDFS-6584
Commit: 7a62515c8628430a163415e42c9526a123db213c
Parents: 71269f7
Author: Allen Wittenauer
Authored: Fri Sep 5 11:31:49 2014 -0700
Committer: Allen Wittenauer
Committed: Fri Sep 5 11:31:49 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop-common/src/main/bin/hadoop-functions.sh | 4 ++--
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs| 15 +++
3 files changed, 12 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a62515c/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index d38fae9..afd1cc7 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -330,6 +330,9 @@ Trunk (Unreleased)
HADOOP-11033. shell scripts ignore JAVA_HOME on OS X. (aw)
+HADOOP-11052. hadoop_verify_secure_prereq's results aren't checked
+in bin/hdfs (aw)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a62515c/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index d430188..1677cc0 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -644,9 +644,9 @@ function hadoop_verify_secure_prereq
# this.
# ${EUID} comes from the shell itself!
- if [[ "${EUID}" -ne 0 ]] || [[ -n "${HADOOP_SECURE_COMMAND}" ]]; then
+ if [[ "${EUID}" -ne 0 ]] && [[ -z "${HADOOP_SECURE_COMMAND}" ]]; then
hadoop_error "ERROR: You must be a privileged in order to run a secure
serice."
-return 1
+exit 1
else
return 0
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a62515c/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index 6872a0e..2300dbf 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -225,14 +225,13 @@ esac
if [[ -n "${secure_service}" ]]; then
HADOOP_SECURE_USER="${secure_user}"
- if hadoop_verify_secure_prereq; then
-hadoop_setup_secure_service
-
priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND-$HOSTNAME}.out"
-
priv_errfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND-$HOSTNAME}.err"
-
priv_pidfile="${HADOOP_PID_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND-$HOSTNAME}.pid"
-
daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
-
daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
- fi
+ hadoop_verify_secure_prereq
+ hadoop_setup_secure_service
+
priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND-$HOSTNAME}.out"
+
priv_errfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND-$HOSTNAME}.err"
+
priv_pidfile="${HADOOP_PID_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND-$HOSTNAME}.pid"
+
daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
+
daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
else
daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
[10/19] git commit: HADOOP-11070. Create MiniKMS for testing. (tucu)
HADOOP-11070. Create MiniKMS for testing. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71c8d735
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71c8d735
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71c8d735
Branch: refs/heads/HDFS-6584
Commit: 71c8d735f5038e3b516947f12180d7568b6979dc
Parents: e6420fe
Author: Alejandro Abdelnur
Authored: Fri Sep 5 14:09:22 2014 -0700
Committer: Alejandro Abdelnur
Committed: Fri Sep 5 21:59:12 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
hadoop-common-project/hadoop-kms/pom.xml| 4 +-
.../hadoop/crypto/key/kms/server/MiniKMS.java | 197 +++
.../hadoop/crypto/key/kms/server/TestKMS.java | 82 +---
4 files changed, 211 insertions(+), 74 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71c8d735/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 88804cd..9aef131 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -507,6 +507,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-11060. Create a CryptoCodec test that verifies interoperability
between the JCE and OpenSSL implementations. (hitliuyi via tucu)
+HADOOP-11070. Create MiniKMS for testing. (tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71c8d735/hadoop-common-project/hadoop-kms/pom.xml
--
diff --git a/hadoop-common-project/hadoop-kms/pom.xml
b/hadoop-common-project/hadoop-kms/pom.xml
index 3bb97c5..629ffda 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -222,9 +222,9 @@
-
+
-
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71c8d735/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
new file mode 100644
index 000..5a6d4c5
--- /dev/null
+++
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.mortbay.jetty.Connector;
+import org.mortbay.jetty.Server;
+import org.mortbay.jetty.security.SslSocketConnector;
+import org.mortbay.jetty.webapp.WebAppContext;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.Writer;
+import java.net.InetAddress;
+import java.net.MalformedURLException;
+import java.net.ServerSocket;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+
+public class MiniKMS {
+
+ private static Server createJettyServer(String keyStore, String password) {
+try {
+ boolean ssl = keyStore != null;
+ InetAddress localhost = InetAddress.getByName("localhost");
+ String host = "localhost";
+ ServerSocket ss = new ServerSocket(0, 50, localhost);
+ int port = ss.getLocalPort();
+ ss.close();
+ Server server = new Server(0);
+ if (!ssl) {
+server.getConnectors()[0].setHost(host);
+server.getConnectors()[0].setPort(port);
+ } else {
+SslSocketConnector c = new SslSocketConnector(
[12/19] git commit: HDFS-6940. Refactoring to allow ConsensusNode implementation. Contributed by Konstantin Shvachko.
HDFS-6940. Refactoring to allow ConsensusNode implementation.
Contributed by Konstantin Shvachko.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88209ce1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88209ce1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88209ce1
Branch: refs/heads/HDFS-6584
Commit: 88209ce181b5ecc55c0ae2bceff4893ab4817e88
Parents: 3b35f81
Author: Konstantin V Shvachko
Authored: Sat Sep 6 12:07:52 2014 -0700
Committer: Konstantin V Shvachko
Committed: Sat Sep 6 12:07:52 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../server/blockmanagement/BlockManager.java| 23 --
.../server/blockmanagement/DatanodeManager.java | 6 ++-
.../server/blockmanagement/HostFileManager.java | 4 ++
.../hdfs/server/namenode/FSNamesystem.java | 46 +++-
.../hdfs/server/namenode/NameNodeAdapter.java | 2 +-
6 files changed, 57 insertions(+), 26 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88209ce1/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 333bdce..4412b30 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -444,6 +444,8 @@ Release 2.6.0 - UNRELEASED
HDFS-6376. Distcp data between two HA clusters requires another
configuration.
(Dave Marion and Haohui Mai via jing9)
+HDFS-6940. Refactoring to allow ConsensusNode implementation. (shv)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88209ce1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 8470680..6176188 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -164,7 +164,7 @@ public class BlockManager {
final BlocksMap blocksMap;
/** Replication thread. */
- final Daemon replicationThread = new Daemon(new ReplicationMonitor());
+ Daemon replicationThread;
/** Store blocks -> datanodedescriptor(s) map of corrupt replicas */
final CorruptReplicasMap corruptReplicas = new CorruptReplicasMap();
@@ -263,6 +263,7 @@ public class BlockManager {
this.namesystem = namesystem;
datanodeManager = new DatanodeManager(this, namesystem, conf);
heartbeatManager = datanodeManager.getHeartbeatManager();
+setReplicationMonitor(new ReplicationMonitor());
final long pendingPeriod = conf.getLong(
DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY,
@@ -394,7 +395,23 @@ public class BlockManager {
lifetimeMin*60*1000L, 0, null, encryptionAlgorithm);
}
}
-
+
+ public long getReplicationRecheckInterval() {
+return replicationRecheckInterval;
+ }
+
+ public AtomicLong excessBlocksCount() {
+return excessBlocksCount;
+ }
+
+ public void clearInvalidateBlocks() {
+invalidateBlocks.clear();
+ }
+
+ void setReplicationMonitor(Runnable replicationMonitor) {
+replicationThread = new Daemon(replicationMonitor);
+ }
+
public void setBlockPoolId(String blockPoolId) {
if (isBlockTokenEnabled()) {
blockTokenSecretManager.setBlockPoolId(blockPoolId);
@@ -1616,7 +1633,7 @@ public class BlockManager {
* If there were any replication requests that timed out, reap them
* and put them back into the neededReplication queue
*/
- private void processPendingReplications() {
+ void processPendingReplications() {
Block[] timedOutItems = pendingReplications.getTimedOutBlocks();
if (timedOutItems != null) {
namesystem.writeLock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88209ce1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index 709f060..55d616f 100644
---
a/hadoop-
[09/19] git commit: HADOOP-11069. KMSClientProvider should use getAuthenticationMethod() to determine if in proxyuser mode or not. (tucu)
HADOOP-11069. KMSClientProvider should use getAuthenticationMethod() to
determine if in proxyuser mode or not. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0f3c19c1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0f3c19c1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0f3c19c1
Branch: refs/heads/HDFS-6584
Commit: 0f3c19c1bb9e341d8aed132ba3eb9e7fc7588306
Parents: 71c8d73
Author: Alejandro Abdelnur
Authored: Fri Sep 5 10:04:07 2014 -0700
Committer: Alejandro Abdelnur
Committed: Fri Sep 5 21:59:12 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt| 3 +++
.../org/apache/hadoop/crypto/key/kms/KMSClientProvider.java| 6 +++---
.../java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java | 6 +++---
3 files changed, 9 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f3c19c1/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 9aef131..c77fddc 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -765,6 +765,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11067. warning message 'ssl.client.truststore.location has not
been set' gets printed for hftp command. (Xiaoyu Yao via Arpit Agarwal)
+HADOOP-11069. KMSClientProvider should use getAuthenticationMethod() to
+determine if in proxyuser mode or not. (tucu)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f3c19c1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index a4e336c..acbe096 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -385,9 +385,9 @@ public class KMSClientProvider extends KeyProvider
implements CryptoExtension,
// if current UGI is different from UGI at constructor time, behave as
// proxyuser
UserGroupInformation currentUgi = UserGroupInformation.getCurrentUser();
- final String doAsUser =
- (loginUgi.getShortUserName().equals(currentUgi.getShortUserName()))
- ? null : currentUgi.getShortUserName();
+ final String doAsUser = (currentUgi.getAuthenticationMethod() ==
+ UserGroupInformation.AuthenticationMethod.PROXY)
+ ? currentUgi.getShortUserName() : null;
// creating the HTTP connection using the current UGI at constructor time
conn = loginUgi.doAs(new PrivilegedExceptionAction() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f3c19c1/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index f381fa0..b921c84 100644
---
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -1157,7 +1157,7 @@ public class TestKMS {
final URI uri = createKMSUri(getKMSUrl());
// proxyuser client using kerberos credentials
-UserGroupInformation clientUgi = UserGroupInformation.
+final UserGroupInformation clientUgi = UserGroupInformation.
loginUserFromKeytabAndReturnUGI("client",
keytab.getAbsolutePath());
clientUgi.doAs(new PrivilegedExceptionAction() {
@Override
@@ -1167,7 +1167,7 @@ public class TestKMS {
// authorized proxyuser
UserGroupInformation fooUgi =
-UserGroupInformation.createRemoteUser("foo");
+UserGroupInformation.createProxyUser("foo", clientUgi);
fooUgi.doAs(new PrivilegedExceptionAction() {
@Override
public Void run() throws Exception {
@@ -1179,7 +1179,7 @@ public class TestKMS {
// unauthorized proxyuser
[16/19] git commit: YARN-2512. Allowed pattern matching for origins in CrossOriginFilter. Contributed by Jonathan Eagles.
YARN-2512. Allowed pattern matching for origins in CrossOriginFilter.
Contributed by Jonathan Eagles.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a092cdf3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a092cdf3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a092cdf3
Branch: refs/heads/HDFS-6584
Commit: a092cdf32de4d752456286a9f4dda533d8a62bca
Parents: a23144f
Author: Zhijie Shen
Authored: Sun Sep 7 17:49:06 2014 -0700
Committer: Zhijie Shen
Committed: Sun Sep 7 17:49:06 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../timeline/webapp/CrossOriginFilter.java | 20 ++-
.../timeline/webapp/TestCrossOriginFilter.java | 21 +++-
3 files changed, 42 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a092cdf3/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index beafc22..ed31479 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -184,6 +184,9 @@ Release 2.6.0 - UNRELEASED
YARN-2508. Cross Origin configuration parameters prefix are not honored
(Mit Desai via jeagles)
+YARN-2512. Allowed pattern matching for origins in CrossOriginFilter.
+(Jonathan Eagles via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a092cdf3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
index d71175f..5a0703d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
@@ -24,6 +24,8 @@ import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import javax.servlet.Filter;
import javax.servlet.FilterChain;
@@ -204,7 +206,23 @@ public class CrossOriginFilter implements Filter {
@VisibleForTesting
boolean isOriginAllowed(String origin) {
-return allowAllOrigins || allowedOrigins.contains(origin);
+if (allowAllOrigins) {
+ return true;
+}
+
+for (String allowedOrigin : allowedOrigins) {
+ if (allowedOrigin.contains("*")) {
+String regex = allowedOrigin.replace(".", "\\.").replace("*", ".*");
+Pattern p = Pattern.compile(regex);
+Matcher m = p.matcher(origin);
+if (m.matches()) {
+ return true;
+}
+ } else if (allowedOrigin.equals(origin)) {
+return true;
+ }
+}
+return false;
}
private boolean areHeadersAllowed(String accessControlRequestHeaders) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a092cdf3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
index f666c21..ccc9bbf 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
@@ -77,7 +77,26 @@ public class TestCrossOriginFilter {
// Object under test
CrossOriginFilter filter = new CrossOriginFilter(
[07/19] git commit: HADOOP-11067 [HDFS-6998]. Fix CHANGES.txt
HADOOP-11067 [HDFS-6998]. Fix CHANGES.txt Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/21c0cdee Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/21c0cdee Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/21c0cdee Branch: refs/heads/HDFS-6584 Commit: 21c0cdeec1034b18ad3a2d5b71941a84bcea5ebe Parents: 0571b45 Author: arp Authored: Fri Sep 5 11:18:20 2014 -0700 Committer: arp Committed: Fri Sep 5 14:29:57 2014 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c0cdee/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index d20bf08..88804cd 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -760,6 +760,9 @@ Release 2.6.0 - UNRELEASED HADOOP-11063. KMS cannot deploy on Windows, because class names are too long. (cnauroth) +HADOOP-11067. warning message 'ssl.client.truststore.location has not +been set' gets printed for hftp command. (Xiaoyu Yao via Arpit Agarwal) + Release 2.5.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c0cdee/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5c4aeea..0772ea6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -610,9 +610,6 @@ Release 2.6.0 - UNRELEASED HDFS-6862. Add missing timeout annotations to tests. (Xiaoyu Yao via Arpit Agarwal) -HDFS-6998. warning message 'ssl.client.truststore.location has not been -set' gets printed for hftp command. (Xiaoyu Yao via Arpit Agarwal) - BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS HDFS-6387. HDFS CLI admin tool for creating & deleting an
[17/19] git commit: YARN-2507. Documented CrossOriginFilter configurations for the timeline server. Contributed by Jonathan Eagles.
YARN-2507. Documented CrossOriginFilter configurations for the timeline server. Contributed by Jonathan Eagles. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/56dc496a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/56dc496a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/56dc496a Branch: refs/heads/HDFS-6584 Commit: 56dc496a1031621d2b701801de4ec29179d75f2e Parents: a092cdf Author: Zhijie Shen Authored: Sun Sep 7 18:22:40 2014 -0700 Committer: Zhijie Shen Committed: Sun Sep 7 18:22:40 2014 -0700 -- hadoop-yarn-project/CHANGES.txt | 3 ++ .../src/site/apt/TimelineServer.apt.vm | 37 2 files changed, 40 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/56dc496a/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index ed31479..ed9de87 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -187,6 +187,9 @@ Release 2.6.0 - UNRELEASED YARN-2512. Allowed pattern matching for origins in CrossOriginFilter. (Jonathan Eagles via zjshen) +YARN-2507. Documented CrossOriginFilter configurations for the timeline +server. (Jonathan Eagles via zjshen) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/56dc496a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm index c704d37..92c7377 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm @@ -102,6 +102,43 @@ YARN Timeline Server yarn.timeline-service.handler-thread-count 10 + + + Enables cross-origin support (CORS) for web services where + cross-origin web response headers are needed. For example, javascript making + a web services request to the timeline server. + yarn.timeline-service.http-cross-origin.enabled + false + + + + Comma separated list of origins that are allowed for web + services needing cross-origin (CORS) support. Wildcards (*) and patterns + allowed + yarn.timeline-service.http-cross-origin.allowed-origins + * + + + + Comma separated list of methods that are allowed for web + services needing cross-origin (CORS) support. + yarn.timeline-service.http-cross-origin.allowed-methods + GET,POST,HEAD + + + + Comma separated list of headers that are allowed for web + services needing cross-origin (CORS) support. + yarn.timeline-service.http-cross-origin.allowed-headers + X-Requested-With,Content-Type,Accept,Origin + + + + The number of seconds a pre-flighted request can be cached + for web services needing cross-origin (CORS) support. + yarn.timeline-service.http-cross-origin.max-age + 1800 + +---+ * Generic-data related Configuration
[14/19] git commit: HDFS-6898. DN must reserve space for a full block when an RBW block is created. (Contributed by Arpit Agarwal)
HDFS-6898. DN must reserve space for a full block when an RBW block is created.
(Contributed by Arpit Agarwal)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d1fa5829
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d1fa5829
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d1fa5829
Branch: refs/heads/HDFS-6584
Commit: d1fa58292e87bc29b4ef1278368c2be938a0afc4
Parents: cbea1b1
Author: arp
Authored: Sat Sep 6 20:02:40 2014 -0700
Committer: arp
Committed: Sat Sep 6 21:04:29 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hadoop/hdfs/protocol/HdfsConstants.java | 2 +-
.../server/datanode/ReplicaBeingWritten.java| 12 +-
.../hdfs/server/datanode/ReplicaInPipeline.java | 33 ++-
.../hdfs/server/datanode/ReplicaInfo.java | 7 +
.../server/datanode/fsdataset/FsVolumeSpi.java | 11 +
.../datanode/fsdataset/impl/BlockPoolSlice.java | 6 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 15 +-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 58 +++-
.../server/datanode/TestDirectoryScanner.java | 8 +
.../fsdataset/impl/TestRbwSpaceReservation.java | 288 +++
.../fsdataset/impl/TestWriteToReplica.java | 2 +-
12 files changed, 423 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1fa5829/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 4412b30..3d43171 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -612,6 +612,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6862. Add missing timeout annotations to tests. (Xiaoyu Yao via
Arpit Agarwal)
+HDFS-6898. DN must reserve space for a full block when an RBW block is
+created. (Arpit Agarwal)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1fa5829/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
index 77fe543..240dcd0 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
@@ -48,7 +48,7 @@ public class HdfsConstants {
"org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol";
- public static final int MIN_BLOCKS_FOR_WRITE = 5;
+ public static final int MIN_BLOCKS_FOR_WRITE = 1;
// Long that indicates "leave current quota unchanged"
public static final long QUOTA_DONT_SET = Long.MAX_VALUE;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1fa5829/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java
index 728dd38..4a89493 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java
@@ -34,10 +34,12 @@ public class ReplicaBeingWritten extends ReplicaInPipeline {
* @param genStamp replica generation stamp
* @param vol volume where replica is located
* @param dir directory path where block and meta files are located
+ * @param bytesToReserve disk space to reserve for this replica, based on
+ * the estimated maximum block length.
*/
public ReplicaBeingWritten(long blockId, long genStamp,
-FsVolumeSpi vol, File dir) {
-super( blockId, genStamp, vol, dir);
+FsVolumeSpi vol, File dir, long bytesToReserve) {
+super(blockId, genStamp, vol, dir, bytesToReserve);
}
/**
@@ -60,10 +62,12 @@ public class ReplicaBeingWritten extends ReplicaInPipeline {
* @param vol volume where replica is located
* @param dir directory path where block and meta files are located
* @param writer a thread that is writing to this replica
+ *
[01/19] git commit: HDFS-6979. hdfs.dll not produce .pdb files. Contributed by Chris Nauroth.
Repository: hadoop Updated Branches: refs/heads/HDFS-6584 22a41dce4 -> f1432e242 HDFS-6979. hdfs.dll not produce .pdb files. Contributed by Chris Nauroth. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fab9bc58 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fab9bc58 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fab9bc58 Branch: refs/heads/HDFS-6584 Commit: fab9bc58ec03ea81cd5ce8a8746a4ee588f7bb08 Parents: 9e941d9 Author: cnauroth Authored: Fri Sep 5 11:03:58 2014 -0700 Committer: cnauroth Committed: Fri Sep 5 11:03:58 2014 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ hadoop-hdfs-project/hadoop-hdfs/pom.xml | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fab9bc58/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5a087d2..680af55 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -605,6 +605,8 @@ Release 2.6.0 - UNRELEASED HDFS-6831. Inconsistency between 'hdfs dfsadmin' and 'hdfs dfsadmin -help'. (Xiaoyu Yao via Arpit Agarwal) +HDFS-6979. hdfs.dll not produce .pdb files. (cnauroth) + BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS HDFS-6387. HDFS CLI admin tool for creating & deleting an http://git-wip-us.apache.org/repos/asf/hadoop/blob/fab9bc58/hadoop-hdfs-project/hadoop-hdfs/pom.xml -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index 2c4ddf6..ecdd1ae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -415,11 +415,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> - + - + @@ -437,7 +437,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> - +
