git commit: HDFS-6940. Refactoring to allow ConsensusNode implementation. Contributed by Konstantin Shvachko.
Repository: hadoop
Updated Branches:
refs/heads/trunk 3b35f8160 -> 88209ce18
HDFS-6940. Refactoring to allow ConsensusNode implementation.
Contributed by Konstantin Shvachko.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88209ce1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88209ce1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88209ce1
Branch: refs/heads/trunk
Commit: 88209ce181b5ecc55c0ae2bceff4893ab4817e88
Parents: 3b35f81
Author: Konstantin V Shvachko
Authored: Sat Sep 6 12:07:52 2014 -0700
Committer: Konstantin V Shvachko
Committed: Sat Sep 6 12:07:52 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../server/blockmanagement/BlockManager.java| 23 --
.../server/blockmanagement/DatanodeManager.java | 6 ++-
.../server/blockmanagement/HostFileManager.java | 4 ++
.../hdfs/server/namenode/FSNamesystem.java | 46 +++-
.../hdfs/server/namenode/NameNodeAdapter.java | 2 +-
6 files changed, 57 insertions(+), 26 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88209ce1/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 333bdce..4412b30 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -444,6 +444,8 @@ Release 2.6.0 - UNRELEASED
HDFS-6376. Distcp data between two HA clusters requires another
configuration.
(Dave Marion and Haohui Mai via jing9)
+HDFS-6940. Refactoring to allow ConsensusNode implementation. (shv)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88209ce1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 8470680..6176188 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -164,7 +164,7 @@ public class BlockManager {
final BlocksMap blocksMap;
/** Replication thread. */
- final Daemon replicationThread = new Daemon(new ReplicationMonitor());
+ Daemon replicationThread;
/** Store blocks -> datanodedescriptor(s) map of corrupt replicas */
final CorruptReplicasMap corruptReplicas = new CorruptReplicasMap();
@@ -263,6 +263,7 @@ public class BlockManager {
this.namesystem = namesystem;
datanodeManager = new DatanodeManager(this, namesystem, conf);
heartbeatManager = datanodeManager.getHeartbeatManager();
+setReplicationMonitor(new ReplicationMonitor());
final long pendingPeriod = conf.getLong(
DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY,
@@ -394,7 +395,23 @@ public class BlockManager {
lifetimeMin*60*1000L, 0, null, encryptionAlgorithm);
}
}
-
+
+ public long getReplicationRecheckInterval() {
+return replicationRecheckInterval;
+ }
+
+ public AtomicLong excessBlocksCount() {
+return excessBlocksCount;
+ }
+
+ public void clearInvalidateBlocks() {
+invalidateBlocks.clear();
+ }
+
+ void setReplicationMonitor(Runnable replicationMonitor) {
+replicationThread = new Daemon(replicationMonitor);
+ }
+
public void setBlockPoolId(String blockPoolId) {
if (isBlockTokenEnabled()) {
blockTokenSecretManager.setBlockPoolId(blockPoolId);
@@ -1616,7 +1633,7 @@ public class BlockManager {
* If there were any replication requests that timed out, reap them
* and put them back into the neededReplication queue
*/
- private void processPendingReplications() {
+ void processPendingReplications() {
Block[] timedOutItems = pendingReplications.getTimedOutBlocks();
if (timedOutItems != null) {
namesystem.writeLock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88209ce1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/bloc
git commit: HDFS-6940. Refactoring to allow ConsensusNode implementation. Contributed by Konstantin Shvachko.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 035112f25 -> 1ea388355
HDFS-6940. Refactoring to allow ConsensusNode implementation.
Contributed by Konstantin Shvachko.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1ea38835
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1ea38835
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1ea38835
Branch: refs/heads/branch-2
Commit: 1ea388355376342e38a2819291a22e83f845a1ef
Parents: 035112f
Author: Konstantin V Shvachko
Authored: Sat Sep 6 12:04:49 2014 -0700
Committer: Konstantin V Shvachko
Committed: Sat Sep 6 12:04:49 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../server/blockmanagement/BlockManager.java| 23 --
.../server/blockmanagement/DatanodeManager.java | 6 ++-
.../server/blockmanagement/HostFileManager.java | 4 ++
.../hdfs/server/namenode/FSNamesystem.java | 46 +++-
.../hdfs/server/namenode/NameNodeAdapter.java | 2 +-
6 files changed, 57 insertions(+), 26 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ea38835/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 5b74293..6a44347 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -188,6 +188,8 @@ Release 2.6.0 - UNRELEASED
HDFS-6376. Distcp data between two HA clusters requires another
configuration.
(Dave Marion and Haohui Mai via jing9)
+HDFS-6940. Refactoring to allow ConsensusNode implementation. (shv)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ea38835/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 5d23c1f..de02de1 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -164,7 +164,7 @@ public class BlockManager {
final BlocksMap blocksMap;
/** Replication thread. */
- final Daemon replicationThread = new Daemon(new ReplicationMonitor());
+ Daemon replicationThread;
/** Store blocks -> datanodedescriptor(s) map of corrupt replicas */
final CorruptReplicasMap corruptReplicas = new CorruptReplicasMap();
@@ -263,6 +263,7 @@ public class BlockManager {
this.namesystem = namesystem;
datanodeManager = new DatanodeManager(this, namesystem, conf);
heartbeatManager = datanodeManager.getHeartbeatManager();
+setReplicationMonitor(new ReplicationMonitor());
final long pendingPeriod = conf.getLong(
DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY,
@@ -394,7 +395,23 @@ public class BlockManager {
lifetimeMin*60*1000L, 0, null, encryptionAlgorithm);
}
}
-
+
+ public long getReplicationRecheckInterval() {
+return replicationRecheckInterval;
+ }
+
+ public AtomicLong excessBlocksCount() {
+return excessBlocksCount;
+ }
+
+ public void clearInvalidateBlocks() {
+invalidateBlocks.clear();
+ }
+
+ void setReplicationMonitor(Runnable replicationMonitor) {
+replicationThread = new Daemon(replicationMonitor);
+ }
+
public void setBlockPoolId(String blockPoolId) {
if (isBlockTokenEnabled()) {
blockTokenSecretManager.setBlockPoolId(blockPoolId);
@@ -1619,7 +1636,7 @@ public class BlockManager {
* If there were any replication requests that timed out, reap them
* and put them back into the neededReplication queue
*/
- private void processPendingReplications() {
+ void processPendingReplications() {
Block[] timedOutItems = pendingReplications.getTimedOutBlocks();
if (timedOutItems != null) {
namesystem.writeLock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ea38835/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/serv
[1/5] git commit: HADOOP-11069. KMSClientProvider should use getAuthenticationMethod() to determine if in proxyuser mode or not. (tucu)
Repository: hadoop
Updated Branches:
refs/heads/HDFS-6581 339d21f27 -> 31bbeaf38
HADOOP-11069. KMSClientProvider should use getAuthenticationMethod() to
determine if in proxyuser mode or not. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0f3c19c1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0f3c19c1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0f3c19c1
Branch: refs/heads/HDFS-6581
Commit: 0f3c19c1bb9e341d8aed132ba3eb9e7fc7588306
Parents: 71c8d73
Author: Alejandro Abdelnur
Authored: Fri Sep 5 10:04:07 2014 -0700
Committer: Alejandro Abdelnur
Committed: Fri Sep 5 21:59:12 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt| 3 +++
.../org/apache/hadoop/crypto/key/kms/KMSClientProvider.java| 6 +++---
.../java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java | 6 +++---
3 files changed, 9 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f3c19c1/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 9aef131..c77fddc 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -765,6 +765,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11067. warning message 'ssl.client.truststore.location has not
been set' gets printed for hftp command. (Xiaoyu Yao via Arpit Agarwal)
+HADOOP-11069. KMSClientProvider should use getAuthenticationMethod() to
+determine if in proxyuser mode or not. (tucu)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f3c19c1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index a4e336c..acbe096 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -385,9 +385,9 @@ public class KMSClientProvider extends KeyProvider
implements CryptoExtension,
// if current UGI is different from UGI at constructor time, behave as
// proxyuser
UserGroupInformation currentUgi = UserGroupInformation.getCurrentUser();
- final String doAsUser =
- (loginUgi.getShortUserName().equals(currentUgi.getShortUserName()))
- ? null : currentUgi.getShortUserName();
+ final String doAsUser = (currentUgi.getAuthenticationMethod() ==
+ UserGroupInformation.AuthenticationMethod.PROXY)
+ ? currentUgi.getShortUserName() : null;
// creating the HTTP connection using the current UGI at constructor time
conn = loginUgi.doAs(new PrivilegedExceptionAction() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f3c19c1/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index f381fa0..b921c84 100644
---
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -1157,7 +1157,7 @@ public class TestKMS {
final URI uri = createKMSUri(getKMSUrl());
// proxyuser client using kerberos credentials
-UserGroupInformation clientUgi = UserGroupInformation.
+final UserGroupInformation clientUgi = UserGroupInformation.
loginUserFromKeytabAndReturnUGI("client",
keytab.getAbsolutePath());
clientUgi.doAs(new PrivilegedExceptionAction() {
@Override
@@ -1167,7 +1167,7 @@ public class TestKMS {
// authorized proxyuser
UserGroupInformation fooUgi =
-UserGroupInformation.createRemoteUser("foo");
+UserGroupInformation.createProxyUser("foo", clientUgi);
fooUgi.doAs(new PrivilegedExceptionAction() {
@Override
public Void run() throws Exception {
@@ -1179,
[2/5] git commit: HADOOP-11070. Create MiniKMS for testing. (tucu)
HADOOP-11070. Create MiniKMS for testing. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71c8d735
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71c8d735
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71c8d735
Branch: refs/heads/HDFS-6581
Commit: 71c8d735f5038e3b516947f12180d7568b6979dc
Parents: e6420fe
Author: Alejandro Abdelnur
Authored: Fri Sep 5 14:09:22 2014 -0700
Committer: Alejandro Abdelnur
Committed: Fri Sep 5 21:59:12 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
hadoop-common-project/hadoop-kms/pom.xml| 4 +-
.../hadoop/crypto/key/kms/server/MiniKMS.java | 197 +++
.../hadoop/crypto/key/kms/server/TestKMS.java | 82 +---
4 files changed, 211 insertions(+), 74 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71c8d735/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 88804cd..9aef131 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -507,6 +507,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-11060. Create a CryptoCodec test that verifies interoperability
between the JCE and OpenSSL implementations. (hitliuyi via tucu)
+HADOOP-11070. Create MiniKMS for testing. (tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71c8d735/hadoop-common-project/hadoop-kms/pom.xml
--
diff --git a/hadoop-common-project/hadoop-kms/pom.xml
b/hadoop-common-project/hadoop-kms/pom.xml
index 3bb97c5..629ffda 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -222,9 +222,9 @@
-
+
-
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71c8d735/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
new file mode 100644
index 000..5a6d4c5
--- /dev/null
+++
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.mortbay.jetty.Connector;
+import org.mortbay.jetty.Server;
+import org.mortbay.jetty.security.SslSocketConnector;
+import org.mortbay.jetty.webapp.WebAppContext;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.Writer;
+import java.net.InetAddress;
+import java.net.MalformedURLException;
+import java.net.ServerSocket;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+
+public class MiniKMS {
+
+ private static Server createJettyServer(String keyStore, String password) {
+try {
+ boolean ssl = keyStore != null;
+ InetAddress localhost = InetAddress.getByName("localhost");
+ String host = "localhost";
+ ServerSocket ss = new ServerSocket(0, 50, localhost);
+ int port = ss.getLocalPort();
+ ss.close();
+ Server server = new Server(0);
+ if (!ssl) {
+server.getConnectors()[0].setHost(host);
+server.getConnectors()[0].setPort(port);
+ } else {
+SslSocketConnector c = new SslSocketConnector(
[4/5] git commit: HDFS-6940. Refactoring to allow ConsensusNode implementation. Contributed by Konstantin Shvachko.
HDFS-6940. Refactoring to allow ConsensusNode implementation.
Contributed by Konstantin Shvachko.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88209ce1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88209ce1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88209ce1
Branch: refs/heads/HDFS-6581
Commit: 88209ce181b5ecc55c0ae2bceff4893ab4817e88
Parents: 3b35f81
Author: Konstantin V Shvachko
Authored: Sat Sep 6 12:07:52 2014 -0700
Committer: Konstantin V Shvachko
Committed: Sat Sep 6 12:07:52 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../server/blockmanagement/BlockManager.java| 23 --
.../server/blockmanagement/DatanodeManager.java | 6 ++-
.../server/blockmanagement/HostFileManager.java | 4 ++
.../hdfs/server/namenode/FSNamesystem.java | 46 +++-
.../hdfs/server/namenode/NameNodeAdapter.java | 2 +-
6 files changed, 57 insertions(+), 26 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88209ce1/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 333bdce..4412b30 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -444,6 +444,8 @@ Release 2.6.0 - UNRELEASED
HDFS-6376. Distcp data between two HA clusters requires another
configuration.
(Dave Marion and Haohui Mai via jing9)
+HDFS-6940. Refactoring to allow ConsensusNode implementation. (shv)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88209ce1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 8470680..6176188 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -164,7 +164,7 @@ public class BlockManager {
final BlocksMap blocksMap;
/** Replication thread. */
- final Daemon replicationThread = new Daemon(new ReplicationMonitor());
+ Daemon replicationThread;
/** Store blocks -> datanodedescriptor(s) map of corrupt replicas */
final CorruptReplicasMap corruptReplicas = new CorruptReplicasMap();
@@ -263,6 +263,7 @@ public class BlockManager {
this.namesystem = namesystem;
datanodeManager = new DatanodeManager(this, namesystem, conf);
heartbeatManager = datanodeManager.getHeartbeatManager();
+setReplicationMonitor(new ReplicationMonitor());
final long pendingPeriod = conf.getLong(
DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY,
@@ -394,7 +395,23 @@ public class BlockManager {
lifetimeMin*60*1000L, 0, null, encryptionAlgorithm);
}
}
-
+
+ public long getReplicationRecheckInterval() {
+return replicationRecheckInterval;
+ }
+
+ public AtomicLong excessBlocksCount() {
+return excessBlocksCount;
+ }
+
+ public void clearInvalidateBlocks() {
+invalidateBlocks.clear();
+ }
+
+ void setReplicationMonitor(Runnable replicationMonitor) {
+replicationThread = new Daemon(replicationMonitor);
+ }
+
public void setBlockPoolId(String blockPoolId) {
if (isBlockTokenEnabled()) {
blockTokenSecretManager.setBlockPoolId(blockPoolId);
@@ -1616,7 +1633,7 @@ public class BlockManager {
* If there were any replication requests that timed out, reap them
* and put them back into the neededReplication queue
*/
- private void processPendingReplications() {
+ void processPendingReplications() {
Block[] timedOutItems = pendingReplications.getTimedOutBlocks();
if (timedOutItems != null) {
namesystem.writeLock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88209ce1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index 709f060..55d616f 100644
---
a/hadoop-
[5/5] git commit: Merge remote-tracking branch 'apache-commit/trunk' into HDFS-6581
Merge remote-tracking branch 'apache-commit/trunk' into HDFS-6581 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/31bbeaf3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/31bbeaf3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/31bbeaf3 Branch: refs/heads/HDFS-6581 Commit: 31bbeaf383a728a0453f5dbe93eed0ab18b5b992 Parents: 339d21f 88209ce Author: arp Authored: Sat Sep 6 12:28:22 2014 -0700 Committer: arp Committed: Sat Sep 6 12:28:22 2014 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 5 + .../crypto/key/kms/KMSClientProvider.java | 6 +- hadoop-common-project/hadoop-kms/pom.xml| 4 +- .../hadoop/crypto/key/kms/server/MiniKMS.java | 197 +++ .../hadoop/crypto/key/kms/server/TestKMS.java | 88 ++--- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 5 + .../java/org/apache/hadoop/hdfs/DFSClient.java | 4 + .../hadoop/hdfs/DistributedFileSystem.java | 24 +++ .../server/blockmanagement/BlockManager.java| 23 ++- .../server/blockmanagement/DatanodeManager.java | 6 +- .../server/blockmanagement/HostFileManager.java | 4 + .../hdfs/server/namenode/FSNamesystem.java | 46 +++-- .../apache/hadoop/hdfs/TestEncryptionZones.java | 43 .../hdfs/server/namenode/NameNodeAdapter.java | 2 +- 14 files changed, 351 insertions(+), 106 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/31bbeaf3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java --
[3/5] git commit: HDFS-6986. DistributedFileSystem must get delegation tokens from configured KeyProvider. (zhz via tucu)
HDFS-6986. DistributedFileSystem must get delegation tokens from configured
KeyProvider. (zhz via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3b35f816
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3b35f816
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3b35f816
Branch: refs/heads/HDFS-6581
Commit: 3b35f81603bbfae119762b50bcb46de70a421368
Parents: 0f3c19c
Author: Alejandro Abdelnur
Authored: Fri Sep 5 22:33:48 2014 -0700
Committer: Alejandro Abdelnur
Committed: Fri Sep 5 22:33:48 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../java/org/apache/hadoop/hdfs/DFSClient.java | 4 ++
.../hadoop/hdfs/DistributedFileSystem.java | 24 +++
.../apache/hadoop/hdfs/TestEncryptionZones.java | 43
4 files changed, 74 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b35f816/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0772ea6..333bdce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -711,6 +711,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6714. TestBlocksScheduledCounter#testBlocksScheduledCounter should
shutdown cluster (vinayakumarb)
+ HDFS-6986. DistributedFileSystem must get delegation tokens from
configured
+ KeyProvider. (zhz via tucu)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b35f816/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 8daf912..e4215f0 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -3084,4 +3084,8 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
DFSHedgedReadMetrics getHedgedReadMetrics() {
return HEDGED_READ_METRIC;
}
+
+ public KeyProviderCryptoExtension getKeyProvider() {
+return provider;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b35f816/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index bf7d62e..dbdf5c1 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -84,8 +84,10 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@@ -1946,6 +1948,28 @@ public class DistributedFileSystem extends FileSystem {
}.resolve(this, absF);
}
+ @Override
+ public Token[] addDelegationTokens(
+ final String renewer, Credentials credentials) throws IOException {
+Token[] tokens = super.addDelegationTokens(renewer, credentials);
+if (dfs.getKeyProvider() != null) {
+ KeyProviderDelegationTokenExtension keyProviderDelegationTokenExtension =
+ KeyProviderDelegationTokenExtension.
+ createKeyProviderDelegationTokenExtension(dfs.getKeyProvider());
+ Token[] kpTokens = keyProviderDelegationTokenExtension.
+ addDelegationTokens(renewer, credentials);
+ if (tokens != null && kpTokens != null) {
+Token[] all = new Token[tokens.length + kpTokens.length];
+System.arraycopy(tokens, 0, all, 0, tokens.length);
+System.arraycopy(kpTokens, 0, all, tokens.length, kpTokens.length);
+tokens = all;
+ } else {
+tokens = (tokens != null) ? tokens : kpTokens;
+ }
+}
+return tokens;
+ }
+
public DFSInotifyEventInputStream getInotifyEve
git commit: HDFS-6997: add more tests for data migration and replicaion.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-6584 ba4fc9317 -> 22a41dce4
HDFS-6997: add more tests for data migration and replicaion.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/22a41dce
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/22a41dce
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/22a41dce
Branch: refs/heads/HDFS-6584
Commit: 22a41dce4af4d5b533ba875b322551db1c152878
Parents: ba4fc93
Author: Tsz-Wo Nicholas Sze
Authored: Sun Sep 7 07:44:28 2014 +0800
Committer: Tsz-Wo Nicholas Sze
Committed: Sun Sep 7 07:44:28 2014 +0800
--
.../server/blockmanagement/BlockManager.java| 2 +-
.../BlockPlacementPolicyDefault.java| 131 +++--
.../blockmanagement/DatanodeDescriptor.java | 52 +-
.../blockmanagement/DatanodeStorageInfo.java| 2 +-
.../apache/hadoop/hdfs/server/mover/Mover.java | 7 +
.../apache/hadoop/hdfs/util/EnumCounters.java | 9 +
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 12 +-
.../hdfs/server/mover/TestStorageMover.java | 477 +--
8 files changed, 579 insertions(+), 113 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22a41dce/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index af83653..956900d 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -2930,7 +2930,7 @@ public class BlockManager {
// Decrement number of blocks scheduled to this datanode.
// for a retry request (of DatanodeProtocol#blockReceivedAndDeleted with
// RECEIVED_BLOCK), we currently also decrease the approximate number.
-node.decrementBlocksScheduled();
+node.decrementBlocksScheduled(storageInfo.getStorageType());
// get the deletion hint node
DatanodeDescriptor delHintNode = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22a41dce/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index 593ea90..a0e6701 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -76,12 +76,6 @@ public class BlockPlacementPolicyDefault extends
BlockPlacementPolicy {
*/
protected int tolerateHeartbeatMultiplier;
- protected BlockPlacementPolicyDefault(Configuration conf, FSClusterStats
stats,
- NetworkTopology clusterMap,
- Host2NodesMap host2datanodeMap) {
-initialize(conf, stats, clusterMap, host2datanodeMap);
- }
-
protected BlockPlacementPolicyDefault() {
}
@@ -174,6 +168,10 @@ public class BlockPlacementPolicyDefault extends
BlockPlacementPolicy {
return getPipeline(writer,
results.toArray(new DatanodeStorageInfo[results.size()]));
} catch (NotEnoughReplicasException nr) {
+ if (LOG.isDebugEnabled()) {
+LOG.debug("Failed to choose with favored nodes (=" + favoredNodes
++ "), disregard favored nodes hint and retry.", nr);
+ }
// Fall back to regular block placement disregarding favored nodes hint
return chooseTarget(src, numOfReplicas, writer,
new ArrayList(numOfReplicas), false,
@@ -291,6 +289,9 @@ public class BlockPlacementPolicyDefault extends
BlockPlacementPolicy {
unavailableStorages, newBlock);
final EnumMap storageTypes =
getRequiredStorageTypes(requiredStorageTypes);
+if (LOG.isTraceEnabled()) {
+ LOG.trace("storageTypes=" + storageTypes);
+}
try {
if ((numOfReplicas = requiredStorageTypes.size()) == 0) {
@@ -337,7 +338,11 @@ public class BlockPlacementPolicyDefault extends
BlockPlacementPolicy {
} catch (NotEnoughReplicasException e) {
final String message = "
git commit: YARN-2519. Credential Provider related unit tests failed on Windows. Contributed by Xiaoyu Yao.
Repository: hadoop
Updated Branches:
refs/heads/trunk 88209ce18 -> cbea1b10e
YARN-2519. Credential Provider related unit tests failed on Windows.
Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cbea1b10
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cbea1b10
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cbea1b10
Branch: refs/heads/trunk
Commit: cbea1b10efd871d04c648af18449dc724685db74
Parents: 88209ce
Author: cnauroth
Authored: Sat Sep 6 20:05:07 2014 -0700
Committer: cnauroth
Committed: Sat Sep 6 20:05:07 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java | 4 +++-
2 files changed, 6 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cbea1b10/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 34a206a..beafc22 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -287,6 +287,9 @@ Release 2.6.0 - UNRELEASED
YARN-2431. NM restart: cgroup is not removed for reacquired containers
(jlowe)
+YARN-2519. Credential Provider related unit tests failed on Windows.
+(Xiaoyu Yao via cnauroth)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cbea1b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
index 18600fd..2bd91b4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertEquals;
import java.io.File;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.http.HttpServer2.Builder;
import org.apache.hadoop.security.alias.CredentialProvider;
@@ -74,8 +75,9 @@ public class TestWebAppUtils {
"target/test-dir"));
Configuration conf = new Configuration();
+final Path jksPath = new Path(testDir.toString(), "test.jks");
final String ourUrl =
-JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks";
+JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
File file = new File(testDir, "test.jks");
file.delete();
git commit: YARN-2519. Credential Provider related unit tests failed on Windows. Contributed by Xiaoyu Yao.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 1ea388355 -> af1d38915
YARN-2519. Credential Provider related unit tests failed on Windows.
Contributed by Xiaoyu Yao.
(cherry picked from commit cbea1b10efd871d04c648af18449dc724685db74)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/af1d3891
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/af1d3891
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/af1d3891
Branch: refs/heads/branch-2
Commit: af1d3891570f23ef391c4b7aed3f6dedbcf9277b
Parents: 1ea3883
Author: cnauroth
Authored: Sat Sep 6 20:05:07 2014 -0700
Committer: cnauroth
Committed: Sat Sep 6 20:06:44 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java | 4 +++-
2 files changed, 6 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af1d3891/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index cf9fe6e..83274b6 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -264,6 +264,9 @@ Release 2.6.0 - UNRELEASED
YARN-2431. NM restart: cgroup is not removed for reacquired containers
(jlowe)
+YARN-2519. Credential Provider related unit tests failed on Windows.
+(Xiaoyu Yao via cnauroth)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af1d3891/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
index 18600fd..2bd91b4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertEquals;
import java.io.File;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.http.HttpServer2.Builder;
import org.apache.hadoop.security.alias.CredentialProvider;
@@ -74,8 +75,9 @@ public class TestWebAppUtils {
"target/test-dir"));
Configuration conf = new Configuration();
+final Path jksPath = new Path(testDir.toString(), "test.jks");
final String ourUrl =
-JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks";
+JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
File file = new File(testDir, "test.jks");
file.delete();
[2/2] git commit: HDFS-6898. DN must reserve space for a full block when an RBW block is created. (Contributed by Arpit Agarwal)
HDFS-6898. DN must reserve space for a full block when an RBW block is created.
(Contributed by Arpit Agarwal)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d1fa5829
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d1fa5829
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d1fa5829
Branch: refs/heads/trunk
Commit: d1fa58292e87bc29b4ef1278368c2be938a0afc4
Parents: cbea1b1
Author: arp
Authored: Sat Sep 6 20:02:40 2014 -0700
Committer: arp
Committed: Sat Sep 6 21:04:29 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hadoop/hdfs/protocol/HdfsConstants.java | 2 +-
.../server/datanode/ReplicaBeingWritten.java| 12 +-
.../hdfs/server/datanode/ReplicaInPipeline.java | 33 ++-
.../hdfs/server/datanode/ReplicaInfo.java | 7 +
.../server/datanode/fsdataset/FsVolumeSpi.java | 11 +
.../datanode/fsdataset/impl/BlockPoolSlice.java | 6 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 15 +-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 58 +++-
.../server/datanode/TestDirectoryScanner.java | 8 +
.../fsdataset/impl/TestRbwSpaceReservation.java | 288 +++
.../fsdataset/impl/TestWriteToReplica.java | 2 +-
12 files changed, 423 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1fa5829/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 4412b30..3d43171 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -612,6 +612,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6862. Add missing timeout annotations to tests. (Xiaoyu Yao via
Arpit Agarwal)
+HDFS-6898. DN must reserve space for a full block when an RBW block is
+created. (Arpit Agarwal)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1fa5829/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
index 77fe543..240dcd0 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
@@ -48,7 +48,7 @@ public class HdfsConstants {
"org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol";
- public static final int MIN_BLOCKS_FOR_WRITE = 5;
+ public static final int MIN_BLOCKS_FOR_WRITE = 1;
// Long that indicates "leave current quota unchanged"
public static final long QUOTA_DONT_SET = Long.MAX_VALUE;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1fa5829/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java
index 728dd38..4a89493 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java
@@ -34,10 +34,12 @@ public class ReplicaBeingWritten extends ReplicaInPipeline {
* @param genStamp replica generation stamp
* @param vol volume where replica is located
* @param dir directory path where block and meta files are located
+ * @param bytesToReserve disk space to reserve for this replica, based on
+ * the estimated maximum block length.
*/
public ReplicaBeingWritten(long blockId, long genStamp,
-FsVolumeSpi vol, File dir) {
-super( blockId, genStamp, vol, dir);
+FsVolumeSpi vol, File dir, long bytesToReserve) {
+super(blockId, genStamp, vol, dir, bytesToReserve);
}
/**
@@ -60,10 +62,12 @@ public class ReplicaBeingWritten extends ReplicaInPipeline {
* @param vol volume where replica is located
* @param dir directory path where block and meta files are located
* @param writer a thread that is writing to this replica
+ * @pa
[1/2] git commit: HDFS-6898. DN must reserve space for a full block when an RBW block is created. (Contributed by Arpit Agarwal)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 af1d38915 -> eba06e61c
refs/heads/trunk cbea1b10e -> d1fa58292
HDFS-6898. DN must reserve space for a full block when an RBW block is created.
(Contributed by Arpit Agarwal)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eba06e61
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eba06e61
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eba06e61
Branch: refs/heads/branch-2
Commit: eba06e61cea1d0ae19826f68ee6b8ca6ba8aadcc
Parents: af1d389
Author: arp
Authored: Sat Sep 6 20:02:40 2014 -0700
Committer: arp
Committed: Sat Sep 6 21:04:21 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hadoop/hdfs/protocol/HdfsConstants.java | 2 +-
.../server/datanode/ReplicaBeingWritten.java| 12 +-
.../hdfs/server/datanode/ReplicaInPipeline.java | 33 ++-
.../hdfs/server/datanode/ReplicaInfo.java | 7 +
.../server/datanode/fsdataset/FsVolumeSpi.java | 11 +
.../datanode/fsdataset/impl/BlockPoolSlice.java | 4 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 15 +-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 56 +++-
.../server/datanode/TestDirectoryScanner.java | 8 +
.../fsdataset/impl/TestRbwSpaceReservation.java | 288 +++
.../fsdataset/impl/TestWriteToReplica.java | 2 +-
12 files changed, 421 insertions(+), 20 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba06e61/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 6a44347..0393164 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -351,6 +351,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6862. Add missing timeout annotations to tests. (Xiaoyu Yao via
Arpit Agarwal)
+HDFS-6898. DN must reserve space for a full block when an RBW block is
+created. (Arpit Agarwal)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba06e61/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
index 24b9fc1..2588167 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
@@ -50,7 +50,7 @@ public class HdfsConstants {
"org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol";
- public static final int MIN_BLOCKS_FOR_WRITE = 5;
+ public static final int MIN_BLOCKS_FOR_WRITE = 1;
// Long that indicates "leave current quota unchanged"
public static final long QUOTA_DONT_SET = Long.MAX_VALUE;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eba06e61/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java
index 728dd38..4a89493 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java
@@ -34,10 +34,12 @@ public class ReplicaBeingWritten extends ReplicaInPipeline {
* @param genStamp replica generation stamp
* @param vol volume where replica is located
* @param dir directory path where block and meta files are located
+ * @param bytesToReserve disk space to reserve for this replica, based on
+ * the estimated maximum block length.
*/
public ReplicaBeingWritten(long blockId, long genStamp,
-FsVolumeSpi vol, File dir) {
-super( blockId, genStamp, vol, dir);
+FsVolumeSpi vol, File dir, long bytesToReserve) {
+super(blockId, genStamp, vol, dir, bytesToReserve);
}
/**
@
git commit: MAPREDUCE-6077. native-task: Remove CustomModule examples in nativetask (seanzhong)
Repository: hadoop Updated Branches: refs/heads/MR-2841 7c91f9b14 -> 52a8b4db9 MAPREDUCE-6077. native-task: Remove CustomModule examples in nativetask (seanzhong) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/52a8b4db Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/52a8b4db Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/52a8b4db Branch: refs/heads/MR-2841 Commit: 52a8b4db92a35598006103c59a2ff93afc5312ee Parents: 7c91f9b Author: Sean Zhong Authored: Sat Sep 6 11:46:07 2014 +0800 Committer: Sean Zhong Committed: Sat Sep 6 11:46:07 2014 +0800 -- .../CHANGES.MAPREDUCE-2841.txt | 1 + .../sdk/example/CustomModule/README.txt | 1 - .../sdk/example/CustomModule/pom.xml| 131 --- .../platform/custom/CustomPlatform.java | 39 -- .../nativetask/serde/custom/CustomWritable.java | 75 --- .../serde/custom/CustomWritableSerializer.java | 33 - .../src/main/native/src/CustomComparator.cpp| 88 - 7 files changed, 1 insertion(+), 367 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/52a8b4db/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt index 539e7be..cfc9412 100644 --- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt +++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt @@ -23,3 +23,4 @@ MAPREDUCE-6055. native-task: findbugs, interface annotations, and other misc cle MAPREDUCE-6067. native-task: fix some counter issues (Binglin Chang) MAPREDUCE-6069. native-task: Lint/style fixes and removal of unused code (todd) MAPREDUCE-6074. native-task: fix release audit, javadoc, javac warnings (todd) +MAPREDUCE-6077. native-task: Remove CustomModule examples in nativetask (seanzhong) http://git-wip-us.apache.org/repos/asf/hadoop/blob/52a8b4db/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/README.txt -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/README.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/README.txt deleted file mode 100644 index 0ad6f1e..000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/README.txt +++ /dev/null @@ -1 +0,0 @@ -This project is depend on hadoop and hadoop-nativetask, so in order to complete the build, you have to specify the hadoop dir first. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/52a8b4db/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/pom.xml -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/pom.xml deleted file mode 100644 index 5bf67c7..000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/pom.xml +++ /dev/null @@ -1,131 +0,0 @@ - - -http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 - http://maven.apache.org/xsd/maven-4.0.0.xsd";> - 4.0.0 - org.apache.hadoop - nativetask-sdk - 2.2.0 - nativetask-sdk - - - - - - - org.apache.hadoop - hadoop-mapreduce-client-nativetask - 2.2.0 - - - - - - -org.apache.maven.plugins -maven-jar-plugin - - -org.apache.maven.plugins -maven-surefire-plugin - - - - - - - - - native - -false - - - - -org.apache.maven.plugins -maven-enforcer-plugin - - -enforce-os - - enforce - - - - - mac - unix - native build only supported on Mac or -Unix - - - true - - - - - -org.apache
[03/16] git commit: HDFS-6979. Fix minor error in CHANGES.txt. Contributed by Chris Nauroth.
HDFS-6979. Fix minor error in CHANGES.txt. Contributed by Chris Nauroth. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b051327a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b051327a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b051327a Branch: refs/heads/MR-2841 Commit: b051327ab6a01774e1dad59e1e547dd16f603789 Parents: fab9bc5 Author: cnauroth Authored: Fri Sep 5 11:07:41 2014 -0700 Committer: cnauroth Committed: Fri Sep 5 11:07:41 2014 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b051327a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 680af55..7b8917b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -605,7 +605,7 @@ Release 2.6.0 - UNRELEASED HDFS-6831. Inconsistency between 'hdfs dfsadmin' and 'hdfs dfsadmin -help'. (Xiaoyu Yao via Arpit Agarwal) -HDFS-6979. hdfs.dll not produce .pdb files. (cnauroth) +HDFS-6979. hdfs.dll does not produce .pdb files. (cnauroth) BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
[02/16] git commit: HDFS-6979. hdfs.dll not produce .pdb files. Contributed by Chris Nauroth.
HDFS-6979. hdfs.dll not produce .pdb files. Contributed by Chris Nauroth. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fab9bc58 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fab9bc58 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fab9bc58 Branch: refs/heads/MR-2841 Commit: fab9bc58ec03ea81cd5ce8a8746a4ee588f7bb08 Parents: 9e941d9 Author: cnauroth Authored: Fri Sep 5 11:03:58 2014 -0700 Committer: cnauroth Committed: Fri Sep 5 11:03:58 2014 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ hadoop-hdfs-project/hadoop-hdfs/pom.xml | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fab9bc58/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5a087d2..680af55 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -605,6 +605,8 @@ Release 2.6.0 - UNRELEASED HDFS-6831. Inconsistency between 'hdfs dfsadmin' and 'hdfs dfsadmin -help'. (Xiaoyu Yao via Arpit Agarwal) +HDFS-6979. hdfs.dll not produce .pdb files. (cnauroth) + BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS HDFS-6387. HDFS CLI admin tool for creating & deleting an http://git-wip-us.apache.org/repos/asf/hadoop/blob/fab9bc58/hadoop-hdfs-project/hadoop-hdfs/pom.xml -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index 2c4ddf6..ecdd1ae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -415,11 +415,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> - + - + @@ -437,7 +437,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> - +
[09/16] git commit: YARN-2508. Cross Origin configuration parameters prefix are not honored (Mit Desai via jeagles)
YARN-2508. Cross Origin configuration parameters prefix are not honored (Mit
Desai via jeagles)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e6420fec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e6420fec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e6420fec
Branch: refs/heads/MR-2841
Commit: e6420fec0af9b8d4f424098688ae4926ff527fcf
Parents: 21c0cde
Author: Jonathan Eagles
Authored: Fri Sep 5 19:42:40 2014 -0500
Committer: Jonathan Eagles
Committed: Fri Sep 5 19:42:40 2014 -0500
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../timeline/webapp/CrossOriginFilterInitializer.java | 12 +++-
.../webapp/TestCrossOriginFilterInitializer.java| 7 ++-
3 files changed, 16 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6420fec/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 1a5ea07..34a206a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -181,6 +181,9 @@ Release 2.6.0 - UNRELEASED
YARN-2511. Allowed all origins by default when CrossOriginFilter is
enabled. (Jonathan Eagles via zjshen)
+YARN-2508. Cross Origin configuration parameters prefix are not honored
+(Mit Desai via jeagles)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6420fec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilterInitializer.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilterInitializer.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilterInitializer.java
index 69e0188..148cc63 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilterInitializer.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilterInitializer.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.timeline.webapp;
+import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
@@ -37,6 +38,15 @@ public class CrossOriginFilterInitializer extends
FilterInitializer {
}
static Map getFilterParameters(Configuration conf) {
-return conf.getValByRegex(PREFIX);
+Map filterParams =
+new HashMap();
+for (Map.Entry entry : conf.getValByRegex(PREFIX)
+.entrySet()) {
+ String name = entry.getKey();
+ String value = entry.getValue();
+ name = name.substring(PREFIX.length());
+ filterParams.put(name, value);
+}
+return filterParams;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6420fec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilterInitializer.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilterInitializer.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilterInitializer.java
index 3199aac..cf26368 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilterInitializer.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilterInitializer.java
@@ -42,11 +42,8 @@ public class TestCrossOriginFilterInitializer {
CrossOriginFilterInitializer.getFilterParameters(conf);
// retrieve values
-String rootvalue =
-filterParameters.get(CrossOriginFilterInitializer.PREFIX +
"rootparam");
-String nestedvalue =
-filterParameters.get(CrossOriginFilterIniti
[13/16] git commit: HDFS-6940. Refactoring to allow ConsensusNode implementation. Contributed by Konstantin Shvachko.
HDFS-6940. Refactoring to allow ConsensusNode implementation.
Contributed by Konstantin Shvachko.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88209ce1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88209ce1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88209ce1
Branch: refs/heads/MR-2841
Commit: 88209ce181b5ecc55c0ae2bceff4893ab4817e88
Parents: 3b35f81
Author: Konstantin V Shvachko
Authored: Sat Sep 6 12:07:52 2014 -0700
Committer: Konstantin V Shvachko
Committed: Sat Sep 6 12:07:52 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../server/blockmanagement/BlockManager.java| 23 --
.../server/blockmanagement/DatanodeManager.java | 6 ++-
.../server/blockmanagement/HostFileManager.java | 4 ++
.../hdfs/server/namenode/FSNamesystem.java | 46 +++-
.../hdfs/server/namenode/NameNodeAdapter.java | 2 +-
6 files changed, 57 insertions(+), 26 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88209ce1/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 333bdce..4412b30 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -444,6 +444,8 @@ Release 2.6.0 - UNRELEASED
HDFS-6376. Distcp data between two HA clusters requires another
configuration.
(Dave Marion and Haohui Mai via jing9)
+HDFS-6940. Refactoring to allow ConsensusNode implementation. (shv)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88209ce1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 8470680..6176188 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -164,7 +164,7 @@ public class BlockManager {
final BlocksMap blocksMap;
/** Replication thread. */
- final Daemon replicationThread = new Daemon(new ReplicationMonitor());
+ Daemon replicationThread;
/** Store blocks -> datanodedescriptor(s) map of corrupt replicas */
final CorruptReplicasMap corruptReplicas = new CorruptReplicasMap();
@@ -263,6 +263,7 @@ public class BlockManager {
this.namesystem = namesystem;
datanodeManager = new DatanodeManager(this, namesystem, conf);
heartbeatManager = datanodeManager.getHeartbeatManager();
+setReplicationMonitor(new ReplicationMonitor());
final long pendingPeriod = conf.getLong(
DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY,
@@ -394,7 +395,23 @@ public class BlockManager {
lifetimeMin*60*1000L, 0, null, encryptionAlgorithm);
}
}
-
+
+ public long getReplicationRecheckInterval() {
+return replicationRecheckInterval;
+ }
+
+ public AtomicLong excessBlocksCount() {
+return excessBlocksCount;
+ }
+
+ public void clearInvalidateBlocks() {
+invalidateBlocks.clear();
+ }
+
+ void setReplicationMonitor(Runnable replicationMonitor) {
+replicationThread = new Daemon(replicationMonitor);
+ }
+
public void setBlockPoolId(String blockPoolId) {
if (isBlockTokenEnabled()) {
blockTokenSecretManager.setBlockPoolId(blockPoolId);
@@ -1616,7 +1633,7 @@ public class BlockManager {
* If there were any replication requests that timed out, reap them
* and put them back into the neededReplication queue
*/
- private void processPendingReplications() {
+ void processPendingReplications() {
Block[] timedOutItems = pendingReplications.getTimedOutBlocks();
if (timedOutItems != null) {
namesystem.writeLock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88209ce1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index 709f060..55d616f 100644
---
a/hadoop-hd
[15/16] git commit: HDFS-6898. DN must reserve space for a full block when an RBW block is created. (Contributed by Arpit Agarwal)
HDFS-6898. DN must reserve space for a full block when an RBW block is created.
(Contributed by Arpit Agarwal)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d1fa5829
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d1fa5829
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d1fa5829
Branch: refs/heads/MR-2841
Commit: d1fa58292e87bc29b4ef1278368c2be938a0afc4
Parents: cbea1b1
Author: arp
Authored: Sat Sep 6 20:02:40 2014 -0700
Committer: arp
Committed: Sat Sep 6 21:04:29 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hadoop/hdfs/protocol/HdfsConstants.java | 2 +-
.../server/datanode/ReplicaBeingWritten.java| 12 +-
.../hdfs/server/datanode/ReplicaInPipeline.java | 33 ++-
.../hdfs/server/datanode/ReplicaInfo.java | 7 +
.../server/datanode/fsdataset/FsVolumeSpi.java | 11 +
.../datanode/fsdataset/impl/BlockPoolSlice.java | 6 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 15 +-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 58 +++-
.../server/datanode/TestDirectoryScanner.java | 8 +
.../fsdataset/impl/TestRbwSpaceReservation.java | 288 +++
.../fsdataset/impl/TestWriteToReplica.java | 2 +-
12 files changed, 423 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1fa5829/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 4412b30..3d43171 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -612,6 +612,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6862. Add missing timeout annotations to tests. (Xiaoyu Yao via
Arpit Agarwal)
+HDFS-6898. DN must reserve space for a full block when an RBW block is
+created. (Arpit Agarwal)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1fa5829/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
index 77fe543..240dcd0 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
@@ -48,7 +48,7 @@ public class HdfsConstants {
"org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol";
- public static final int MIN_BLOCKS_FOR_WRITE = 5;
+ public static final int MIN_BLOCKS_FOR_WRITE = 1;
// Long that indicates "leave current quota unchanged"
public static final long QUOTA_DONT_SET = Long.MAX_VALUE;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1fa5829/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java
index 728dd38..4a89493 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java
@@ -34,10 +34,12 @@ public class ReplicaBeingWritten extends ReplicaInPipeline {
* @param genStamp replica generation stamp
* @param vol volume where replica is located
* @param dir directory path where block and meta files are located
+ * @param bytesToReserve disk space to reserve for this replica, based on
+ * the estimated maximum block length.
*/
public ReplicaBeingWritten(long blockId, long genStamp,
-FsVolumeSpi vol, File dir) {
-super( blockId, genStamp, vol, dir);
+FsVolumeSpi vol, File dir, long bytesToReserve) {
+super(blockId, genStamp, vol, dir, bytesToReserve);
}
/**
@@ -60,10 +62,12 @@ public class ReplicaBeingWritten extends ReplicaInPipeline {
* @param vol volume where replica is located
* @param dir directory path where block and meta files are located
* @param writer a thread that is writing to this replica
+ * @
[11/16] git commit: HADOOP-11069. KMSClientProvider should use getAuthenticationMethod() to determine if in proxyuser mode or not. (tucu)
HADOOP-11069. KMSClientProvider should use getAuthenticationMethod() to
determine if in proxyuser mode or not. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0f3c19c1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0f3c19c1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0f3c19c1
Branch: refs/heads/MR-2841
Commit: 0f3c19c1bb9e341d8aed132ba3eb9e7fc7588306
Parents: 71c8d73
Author: Alejandro Abdelnur
Authored: Fri Sep 5 10:04:07 2014 -0700
Committer: Alejandro Abdelnur
Committed: Fri Sep 5 21:59:12 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt| 3 +++
.../org/apache/hadoop/crypto/key/kms/KMSClientProvider.java| 6 +++---
.../java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java | 6 +++---
3 files changed, 9 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f3c19c1/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 9aef131..c77fddc 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -765,6 +765,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11067. warning message 'ssl.client.truststore.location has not
been set' gets printed for hftp command. (Xiaoyu Yao via Arpit Agarwal)
+HADOOP-11069. KMSClientProvider should use getAuthenticationMethod() to
+determine if in proxyuser mode or not. (tucu)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f3c19c1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index a4e336c..acbe096 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -385,9 +385,9 @@ public class KMSClientProvider extends KeyProvider
implements CryptoExtension,
// if current UGI is different from UGI at constructor time, behave as
// proxyuser
UserGroupInformation currentUgi = UserGroupInformation.getCurrentUser();
- final String doAsUser =
- (loginUgi.getShortUserName().equals(currentUgi.getShortUserName()))
- ? null : currentUgi.getShortUserName();
+ final String doAsUser = (currentUgi.getAuthenticationMethod() ==
+ UserGroupInformation.AuthenticationMethod.PROXY)
+ ? currentUgi.getShortUserName() : null;
// creating the HTTP connection using the current UGI at constructor time
conn = loginUgi.doAs(new PrivilegedExceptionAction() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f3c19c1/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index f381fa0..b921c84 100644
---
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -1157,7 +1157,7 @@ public class TestKMS {
final URI uri = createKMSUri(getKMSUrl());
// proxyuser client using kerberos credentials
-UserGroupInformation clientUgi = UserGroupInformation.
+final UserGroupInformation clientUgi = UserGroupInformation.
loginUserFromKeytabAndReturnUGI("client",
keytab.getAbsolutePath());
clientUgi.doAs(new PrivilegedExceptionAction() {
@Override
@@ -1167,7 +1167,7 @@ public class TestKMS {
// authorized proxyuser
UserGroupInformation fooUgi =
-UserGroupInformation.createRemoteUser("foo");
+UserGroupInformation.createProxyUser("foo", clientUgi);
fooUgi.doAs(new PrivilegedExceptionAction() {
@Override
public Void run() throws Exception {
@@ -1179,7 +1179,7 @@ public class TestKMS {
// unauthorized proxyuser
[12/16] git commit: HDFS-6986. DistributedFileSystem must get delegation tokens from configured KeyProvider. (zhz via tucu)
HDFS-6986. DistributedFileSystem must get delegation tokens from configured
KeyProvider. (zhz via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3b35f816
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3b35f816
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3b35f816
Branch: refs/heads/MR-2841
Commit: 3b35f81603bbfae119762b50bcb46de70a421368
Parents: 0f3c19c
Author: Alejandro Abdelnur
Authored: Fri Sep 5 22:33:48 2014 -0700
Committer: Alejandro Abdelnur
Committed: Fri Sep 5 22:33:48 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../java/org/apache/hadoop/hdfs/DFSClient.java | 4 ++
.../hadoop/hdfs/DistributedFileSystem.java | 24 +++
.../apache/hadoop/hdfs/TestEncryptionZones.java | 43
4 files changed, 74 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b35f816/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0772ea6..333bdce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -711,6 +711,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6714. TestBlocksScheduledCounter#testBlocksScheduledCounter should
shutdown cluster (vinayakumarb)
+ HDFS-6986. DistributedFileSystem must get delegation tokens from
configured
+ KeyProvider. (zhz via tucu)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b35f816/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 8daf912..e4215f0 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -3084,4 +3084,8 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
DFSHedgedReadMetrics getHedgedReadMetrics() {
return HEDGED_READ_METRIC;
}
+
+ public KeyProviderCryptoExtension getKeyProvider() {
+return provider;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b35f816/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index bf7d62e..dbdf5c1 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -84,8 +84,10 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@@ -1946,6 +1948,28 @@ public class DistributedFileSystem extends FileSystem {
}.resolve(this, absF);
}
+ @Override
+ public Token[] addDelegationTokens(
+ final String renewer, Credentials credentials) throws IOException {
+Token[] tokens = super.addDelegationTokens(renewer, credentials);
+if (dfs.getKeyProvider() != null) {
+ KeyProviderDelegationTokenExtension keyProviderDelegationTokenExtension =
+ KeyProviderDelegationTokenExtension.
+ createKeyProviderDelegationTokenExtension(dfs.getKeyProvider());
+ Token[] kpTokens = keyProviderDelegationTokenExtension.
+ addDelegationTokens(renewer, credentials);
+ if (tokens != null && kpTokens != null) {
+Token[] all = new Token[tokens.length + kpTokens.length];
+System.arraycopy(tokens, 0, all, 0, tokens.length);
+System.arraycopy(kpTokens, 0, all, tokens.length, kpTokens.length);
+tokens = all;
+ } else {
+tokens = (tokens != null) ? tokens : kpTokens;
+ }
+}
+return tokens;
+ }
+
public DFSInotifyEventInputStream getInotifyEvent
[05/16] git commit: HDFS-6998. warning message 'ssl.client.truststore.location has not been set' gets printed for hftp command. (Contributed by Xiaoyu Yao)
HDFS-6998. warning message 'ssl.client.truststore.location has not been set'
gets printed for hftp command. (Contributed by Xiaoyu Yao)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71269f70
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71269f70
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71269f70
Branch: refs/heads/MR-2841
Commit: 71269f70971dc7aa7bcb5e78b19cb3f04fdaa2f4
Parents: 9609b73
Author: arp
Authored: Fri Sep 5 11:14:10 2014 -0700
Committer: arp
Committed: Fri Sep 5 11:14:10 2014 -0700
--
.../org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
2 files changed, 4 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71269f70/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java
index aabb815..4b81e17 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java
@@ -212,7 +212,7 @@ public class FileBasedKeyStoresFactory implements
KeyStoresFactory {
LOG.debug(mode.toString() + " Loaded TrustStore: " + truststoreLocation);
trustManagers = new TrustManager[]{trustManager};
} else {
- LOG.warn("The property '" + locationProperty + "' has not been set, " +
+ LOG.debug("The property '" + locationProperty + "' has not been set, " +
"no TrustStore will be loaded");
trustManagers = null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71269f70/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0772ea6..5c4aeea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -610,6 +610,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6862. Add missing timeout annotations to tests. (Xiaoyu Yao via
Arpit Agarwal)
+HDFS-6998. warning message 'ssl.client.truststore.location has not been
+set' gets printed for hftp command. (Xiaoyu Yao via Arpit Agarwal)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
[08/16] git commit: HADOOP-11067 [HDFS-6998]. Fix CHANGES.txt
HADOOP-11067 [HDFS-6998]. Fix CHANGES.txt Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/21c0cdee Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/21c0cdee Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/21c0cdee Branch: refs/heads/MR-2841 Commit: 21c0cdeec1034b18ad3a2d5b71941a84bcea5ebe Parents: 0571b45 Author: arp Authored: Fri Sep 5 11:18:20 2014 -0700 Committer: arp Committed: Fri Sep 5 14:29:57 2014 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c0cdee/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index d20bf08..88804cd 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -760,6 +760,9 @@ Release 2.6.0 - UNRELEASED HADOOP-11063. KMS cannot deploy on Windows, because class names are too long. (cnauroth) +HADOOP-11067. warning message 'ssl.client.truststore.location has not +been set' gets printed for hftp command. (Xiaoyu Yao via Arpit Agarwal) + Release 2.5.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c0cdee/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5c4aeea..0772ea6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -610,9 +610,6 @@ Release 2.6.0 - UNRELEASED HDFS-6862. Add missing timeout annotations to tests. (Xiaoyu Yao via Arpit Agarwal) -HDFS-6998. warning message 'ssl.client.truststore.location has not been -set' gets printed for hftp command. (Xiaoyu Yao via Arpit Agarwal) - BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS HDFS-6387. HDFS CLI admin tool for creating & deleting an
[10/16] git commit: HADOOP-11070. Create MiniKMS for testing. (tucu)
HADOOP-11070. Create MiniKMS for testing. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71c8d735
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71c8d735
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71c8d735
Branch: refs/heads/MR-2841
Commit: 71c8d735f5038e3b516947f12180d7568b6979dc
Parents: e6420fe
Author: Alejandro Abdelnur
Authored: Fri Sep 5 14:09:22 2014 -0700
Committer: Alejandro Abdelnur
Committed: Fri Sep 5 21:59:12 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
hadoop-common-project/hadoop-kms/pom.xml| 4 +-
.../hadoop/crypto/key/kms/server/MiniKMS.java | 197 +++
.../hadoop/crypto/key/kms/server/TestKMS.java | 82 +---
4 files changed, 211 insertions(+), 74 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71c8d735/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 88804cd..9aef131 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -507,6 +507,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-11060. Create a CryptoCodec test that verifies interoperability
between the JCE and OpenSSL implementations. (hitliuyi via tucu)
+HADOOP-11070. Create MiniKMS for testing. (tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71c8d735/hadoop-common-project/hadoop-kms/pom.xml
--
diff --git a/hadoop-common-project/hadoop-kms/pom.xml
b/hadoop-common-project/hadoop-kms/pom.xml
index 3bb97c5..629ffda 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -222,9 +222,9 @@
-
+
-
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71c8d735/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
new file mode 100644
index 000..5a6d4c5
--- /dev/null
+++
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.mortbay.jetty.Connector;
+import org.mortbay.jetty.Server;
+import org.mortbay.jetty.security.SslSocketConnector;
+import org.mortbay.jetty.webapp.WebAppContext;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.Writer;
+import java.net.InetAddress;
+import java.net.MalformedURLException;
+import java.net.ServerSocket;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+
+public class MiniKMS {
+
+ private static Server createJettyServer(String keyStore, String password) {
+try {
+ boolean ssl = keyStore != null;
+ InetAddress localhost = InetAddress.getByName("localhost");
+ String host = "localhost";
+ ServerSocket ss = new ServerSocket(0, 50, localhost);
+ int port = ss.getLocalPort();
+ ss.close();
+ Server server = new Server(0);
+ if (!ssl) {
+server.getConnectors()[0].setHost(host);
+server.getConnectors()[0].setPort(port);
+ } else {
+SslSocketConnector c = new SslSocketConnector();
[01/16] git commit: HDFS-6831. Inconsistency between 'hdfs dfsadmin' and 'hdfs dfsadmin -help'. (Contributed by Xiaoyu Yao)
Repository: hadoop
Updated Branches:
refs/heads/MR-2841 52a8b4db9 -> b160707ac
HDFS-6831. Inconsistency between 'hdfs dfsadmin' and 'hdfs dfsadmin -help'.
(Contributed by Xiaoyu Yao)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9e941d9f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9e941d9f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9e941d9f
Branch: refs/heads/MR-2841
Commit: 9e941d9f99168cae01f8d50622a616fc26c196d9
Parents: c6107f5
Author: arp
Authored: Fri Sep 5 10:45:54 2014 -0700
Committer: arp
Committed: Fri Sep 5 10:45:54 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 175 ---
.../java/org/apache/hadoop/tools/TestTools.java | 2 +-
3 files changed, 81 insertions(+), 99 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e941d9f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index d4059de..5a087d2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -602,6 +602,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6996. SnapshotDiff report can hit IndexOutOfBoundsException when there
are nested renamed directory/file. (jing9)
+HDFS-6831. Inconsistency between 'hdfs dfsadmin' and 'hdfs dfsadmin -help'.
+(Xiaoyu Yao via Arpit Agarwal)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e941d9f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
index ad7be18..13ccae5 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
@@ -353,6 +353,40 @@ public class DFSAdmin extends FsShell {
}
/**
+ * Common usage summary shared between "hdfs dfsadmin -help" and
+ * "hdfs dfsadmin"
+ */
+ private static final String commonUsageSummary =
+"\t[-report [-live] [-dead] [-decommissioning]]\n" +
+"\t[-safemode ]\n" +
+"\t[-saveNamespace]\n" +
+"\t[-rollEdits]\n" +
+"\t[-restoreFailedStorage true|false|check]\n" +
+"\t[-refreshNodes]\n" +
+"\t[" + SetQuotaCommand.USAGE + "]\n" +
+"\t[" + ClearQuotaCommand.USAGE +"]\n" +
+"\t[" + SetSpaceQuotaCommand.USAGE + "]\n" +
+"\t[" + ClearSpaceQuotaCommand.USAGE +"]\n" +
+"\t[-finalizeUpgrade]\n" +
+"\t[" + RollingUpgradeCommand.USAGE +"]\n" +
+"\t[-refreshServiceAcl]\n" +
+"\t[-refreshUserToGroupsMappings]\n" +
+"\t[-refreshSuperUserGroupsConfiguration]\n" +
+"\t[-refreshCallQueue]\n" +
+"\t[-refresh [arg1..argn]\n" +
+"\t[-printTopology]\n" +
+"\t[-refreshNamenodes datanode_host:ipc_port]\n"+
+"\t[-deleteBlockPool datanode_host:ipc_port blockpoolId [force]]\n"+
+"\t[-setBalancerBandwidth ]\n" +
+"\t[-fetchImage ]\n" +
+"\t[-allowSnapshot ]\n" +
+"\t[-disallowSnapshot ]\n" +
+"\t[-shutdownDatanode [upgrade]]\n" +
+"\t[-getDatanodeInfo ]\n" +
+"\t[-metasave filename]\n" +
+"\t[-help [cmd]]\n";
+
+ /**
* Construct a DFSAdmin object.
*/
public DFSAdmin() {
@@ -589,7 +623,7 @@ public class DFSAdmin extends FsShell {
/**
* Command to ask the namenode to save the namespace.
- * Usage: java DFSAdmin -saveNamespace
+ * Usage: hdfs dfsadmin -saveNamespace
* @exception IOException
* @see org.apache.hadoop.hdfs.protocol.ClientProtocol#saveNamespace()
*/
@@ -630,7 +664,7 @@ public class DFSAdmin extends FsShell {
/**
* Command to enable/disable/check restoring of failed storage replicas in
the namenode.
- * Usage: java DFSAdmin -restoreFailedStorage true|false|check
+ * Usage: hdfs dfsadmin -restoreFailedStorage true|false|check
* @exception IOException
* @see
org.apache.hadoop.hdfs.protocol.ClientProtocol#restoreFailedStorage(String arg)
*/
@@ -668,7 +702,7 @@ public class DFSAdmin extends FsShell {
/**
* Command to ask the namenode to reread the hosts and excluded hosts
* file.
- * Usage: java DFSAdmin -refreshNodes
+ * Usage: hdfs dfsadmin -refreshNodes
* @exception IOException
*/
public int refreshNodes(
[04/16] git commit: HDFS-6862. Add missing timeout annotations to tests. (Contributed by Xiaoyu Yao)
HDFS-6862. Add missing timeout annotations to tests. (Contributed by Xiaoyu Yao)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9609b730
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9609b730
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9609b730
Branch: refs/heads/MR-2841
Commit: 9609b7303a98c8eff676c5a086b08b1ca9ab777c
Parents: b051327
Author: arp
Authored: Fri Sep 5 11:08:03 2014 -0700
Committer: arp
Committed: Fri Sep 5 11:10:58 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../apache/hadoop/hdfs/TestHDFSServerPorts.java | 24
.../TestValidateConfigurationSettings.java | 16 ++---
.../namenode/ha/TestDelegationTokensWithHA.java | 63
.../hdfs/server/namenode/ha/TestHAMetrics.java | 10 ++--
.../namenode/ha/TestHAStateTransitions.java | 52 +++-
.../namenode/ha/TestStandbyCheckpoints.java | 54 +++--
7 files changed, 97 insertions(+), 125 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9609b730/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 7b8917b..0772ea6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -607,6 +607,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6979. hdfs.dll does not produce .pdb files. (cnauroth)
+HDFS-6862. Add missing timeout annotations to tests. (Xiaoyu Yao via
+Arpit Agarwal)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9609b730/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSServerPorts.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSServerPorts.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSServerPorts.java
index 59d1615..ce8a4e7 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSServerPorts.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSServerPorts.java
@@ -17,14 +17,6 @@
*/
package org.apache.hadoop.hdfs;
-import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.UnknownHostException;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -39,6 +31,14 @@ import org.apache.hadoop.net.DNS;
import org.apache.hadoop.test.PathUtils;
import org.junit.Test;
+import java.io.File;
+import java.io.IOException;
+import java.net.UnknownHostException;
+
+import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
/**
* This test checks correctness of port usage by hdfs components:
* NameNode, DataNode, SecondaryNamenode and BackupNode.
@@ -245,7 +245,7 @@ public class TestHDFSServerPorts {
return true;
}
- @Test
+ @Test(timeout = 30)
public void testNameNodePorts() throws Exception {
runTestNameNodePorts(false);
runTestNameNodePorts(true);
@@ -296,7 +296,7 @@ public class TestHDFSServerPorts {
/**
* Verify datanode port usage.
*/
- @Test
+ @Test(timeout = 30)
public void testDataNodePorts() throws Exception {
NameNode nn = null;
try {
@@ -332,7 +332,7 @@ public class TestHDFSServerPorts {
/**
* Verify secondary namenode port usage.
*/
- @Test
+ @Test(timeout = 30)
public void testSecondaryNodePorts() throws Exception {
NameNode nn = null;
try {
@@ -361,7 +361,7 @@ public class TestHDFSServerPorts {
/**
* Verify BackupNode port usage.
*/
- @Test
+@Test(timeout = 30)
public void testBackupNodePorts() throws Exception {
NameNode nn = null;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9609b730/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestValidateConfigurationSettings.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestValidateConfigurationSettings.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/s
[07/16] git commit: HADOOP-11065. Rat check should exclude **/build/**. (kasha)
HADOOP-11065. Rat check should exclude **/build/**. (kasha) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0571b456 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0571b456 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0571b456 Branch: refs/heads/MR-2841 Commit: 0571b4561bad7e0230920e52d3758a3658fcf20d Parents: 7a62515 Author: Karthik Kambatla Authored: Fri Sep 5 13:08:59 2014 -0700 Committer: Karthik Kambatla Committed: Fri Sep 5 13:09:10 2014 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ pom.xml | 1 + 2 files changed, 3 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0571b456/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index afd1cc7..d20bf08 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -777,6 +777,8 @@ Release 2.5.1 - UNRELEASED HADOOP-11001. Fix test-patch to work with the git repo. (kasha) +HADOOP-11065. Rat check should exclude "**/build/**". (kasha) + Release 2.5.0 - 2014-08-11 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/0571b456/pom.xml -- diff --git a/pom.xml b/pom.xml index a4f8241..5cc30c2 100644 --- a/pom.xml +++ b/pom.xml @@ -324,6 +324,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs .gitignore .git/** .idea/** + **/build/**
[14/16] git commit: YARN-2519. Credential Provider related unit tests failed on Windows. Contributed by Xiaoyu Yao.
YARN-2519. Credential Provider related unit tests failed on Windows.
Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cbea1b10
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cbea1b10
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cbea1b10
Branch: refs/heads/MR-2841
Commit: cbea1b10efd871d04c648af18449dc724685db74
Parents: 88209ce
Author: cnauroth
Authored: Sat Sep 6 20:05:07 2014 -0700
Committer: cnauroth
Committed: Sat Sep 6 20:05:07 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java | 4 +++-
2 files changed, 6 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cbea1b10/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 34a206a..beafc22 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -287,6 +287,9 @@ Release 2.6.0 - UNRELEASED
YARN-2431. NM restart: cgroup is not removed for reacquired containers
(jlowe)
+YARN-2519. Credential Provider related unit tests failed on Windows.
+(Xiaoyu Yao via cnauroth)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cbea1b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
index 18600fd..2bd91b4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertEquals;
import java.io.File;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.http.HttpServer2.Builder;
import org.apache.hadoop.security.alias.CredentialProvider;
@@ -74,8 +75,9 @@ public class TestWebAppUtils {
"target/test-dir"));
Configuration conf = new Configuration();
+final Path jksPath = new Path(testDir.toString(), "test.jks");
final String ourUrl =
-JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks";
+JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
File file = new File(testDir, "test.jks");
file.delete();
[06/16] git commit: HADOOP-11052. hadoop_verify_secure_prereq's results aren't checked in bin/hdfs (aw)
HADOOP-11052. hadoop_verify_secure_prereq's results aren't checked in bin/hdfs
(aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7a62515c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7a62515c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7a62515c
Branch: refs/heads/MR-2841
Commit: 7a62515c8628430a163415e42c9526a123db213c
Parents: 71269f7
Author: Allen Wittenauer
Authored: Fri Sep 5 11:31:49 2014 -0700
Committer: Allen Wittenauer
Committed: Fri Sep 5 11:31:49 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop-common/src/main/bin/hadoop-functions.sh | 4 ++--
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs| 15 +++
3 files changed, 12 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a62515c/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index d38fae9..afd1cc7 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -330,6 +330,9 @@ Trunk (Unreleased)
HADOOP-11033. shell scripts ignore JAVA_HOME on OS X. (aw)
+HADOOP-11052. hadoop_verify_secure_prereq's results aren't checked
+in bin/hdfs (aw)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a62515c/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index d430188..1677cc0 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -644,9 +644,9 @@ function hadoop_verify_secure_prereq
# this.
# ${EUID} comes from the shell itself!
- if [[ "${EUID}" -ne 0 ]] || [[ -n "${HADOOP_SECURE_COMMAND}" ]]; then
+ if [[ "${EUID}" -ne 0 ]] && [[ -z "${HADOOP_SECURE_COMMAND}" ]]; then
hadoop_error "ERROR: You must be a privileged in order to run a secure
serice."
-return 1
+exit 1
else
return 0
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a62515c/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index 6872a0e..2300dbf 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -225,14 +225,13 @@ esac
if [[ -n "${secure_service}" ]]; then
HADOOP_SECURE_USER="${secure_user}"
- if hadoop_verify_secure_prereq; then
-hadoop_setup_secure_service
-
priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND-$HOSTNAME}.out"
-
priv_errfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND-$HOSTNAME}.err"
-
priv_pidfile="${HADOOP_PID_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND-$HOSTNAME}.pid"
-
daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
-
daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
- fi
+ hadoop_verify_secure_prereq
+ hadoop_setup_secure_service
+
priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND-$HOSTNAME}.out"
+
priv_errfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND-$HOSTNAME}.err"
+
priv_pidfile="${HADOOP_PID_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND-$HOSTNAME}.pid"
+
daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
+
daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
else
daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
[16/16] git commit: Merge branch 'trunk' into MR-2841
Merge branch 'trunk' into MR-2841 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b160707a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b160707a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b160707a Branch: refs/heads/MR-2841 Commit: b160707ace7900f360fa914cd291675c1d9703ec Parents: 52a8b4d d1fa582 Author: Sean Zhong Authored: Sun Sep 7 13:01:27 2014 +0800 Committer: Sean Zhong Committed: Sun Sep 7 13:01:27 2014 +0800 -- hadoop-common-project/hadoop-common/CHANGES.txt | 13 + .../src/main/bin/hadoop-functions.sh| 4 +- .../crypto/key/kms/KMSClientProvider.java | 6 +- .../security/ssl/FileBasedKeyStoresFactory.java | 2 +- hadoop-common-project/hadoop-kms/pom.xml| 4 +- .../hadoop/crypto/key/kms/server/MiniKMS.java | 197 + .../hadoop/crypto/key/kms/server/TestKMS.java | 88 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 16 ++ hadoop-hdfs-project/hadoop-hdfs/pom.xml | 6 +- .../hadoop-hdfs/src/main/bin/hdfs | 15 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 4 + .../hadoop/hdfs/DistributedFileSystem.java | 24 ++ .../hadoop/hdfs/protocol/HdfsConstants.java | 2 +- .../server/blockmanagement/BlockManager.java| 23 +- .../server/blockmanagement/DatanodeManager.java | 6 +- .../server/blockmanagement/HostFileManager.java | 4 + .../server/datanode/ReplicaBeingWritten.java| 12 +- .../hdfs/server/datanode/ReplicaInPipeline.java | 33 ++- .../hdfs/server/datanode/ReplicaInfo.java | 7 + .../server/datanode/fsdataset/FsVolumeSpi.java | 11 + .../datanode/fsdataset/impl/BlockPoolSlice.java | 6 +- .../datanode/fsdataset/impl/FsDatasetImpl.java | 15 +- .../datanode/fsdataset/impl/FsVolumeImpl.java | 58 +++- .../hdfs/server/namenode/FSNamesystem.java | 46 +-- .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 175 +-- .../apache/hadoop/hdfs/TestEncryptionZones.java | 43 +++ .../apache/hadoop/hdfs/TestHDFSServerPorts.java | 24 +- .../server/datanode/TestDirectoryScanner.java | 8 + .../fsdataset/impl/TestRbwSpaceReservation.java | 288 +++ .../fsdataset/impl/TestWriteToReplica.java | 2 +- .../hdfs/server/namenode/NameNodeAdapter.java | 2 +- .../TestValidateConfigurationSettings.java | 16 +- .../namenode/ha/TestDelegationTokensWithHA.java | 63 ++-- .../hdfs/server/namenode/ha/TestHAMetrics.java | 10 +- .../namenode/ha/TestHAStateTransitions.java | 52 ++-- .../namenode/ha/TestStandbyCheckpoints.java | 54 ++-- .../java/org/apache/hadoop/tools/TestTools.java | 2 +- hadoop-yarn-project/CHANGES.txt | 6 + .../yarn/webapp/util/TestWebAppUtils.java | 4 +- .../webapp/CrossOriginFilterInitializer.java| 12 +- .../TestCrossOriginFilterInitializer.java | 7 +- pom.xml | 1 + 42 files changed, 998 insertions(+), 373 deletions(-) --
