[hadoop] branch trunk updated: HADOOP-17111. Replace Guava Optional with Java8+ Optional. Contributed by Ahmed Hussein.
This is an automated email from the ASF dual-hosted git repository.
aajisaka pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 639acb6 HADOOP-17111. Replace Guava Optional with Java8+ Optional.
Contributed by Ahmed Hussein.
639acb6 is described below
commit 639acb6d8921127cde3174a302f2e3d71b44f052
Author: Akira Ajisaka
AuthorDate: Mon Jul 6 16:08:36 2020 +0900
HADOOP-17111. Replace Guava Optional with Java8+ Optional. Contributed by
Ahmed Hussein.
---
.../src/main/resources/checkstyle/checkstyle.xml | 7 ++-
.../nodemanager/DefaultContainerExecutor.java | 19 +++
.../server/nodemanager/LinuxContainerExecutor.java | 58 +++---
.../recovery/TestZKRMStateStorePerf.java | 23 -
4 files changed, 66 insertions(+), 41 deletions(-)
diff --git a/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml
b/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml
index 8f3d3f1..54a5943 100644
--- a/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml
+++ b/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml
@@ -119,7 +119,12 @@
-
+
+
+
+
+
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index c5fc481..b8f94b8 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -21,12 +21,7 @@ package org.apache.hadoop.yarn.server.nodemanager;
import static org.apache.hadoop.fs.CreateFlag.CREATE;
import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
-import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair;
-import
org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
-import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerExecContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
+import com.google.common.annotations.VisibleForTesting;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
@@ -38,7 +33,7 @@ import java.util.Arrays;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
-
+import java.util.Optional;
import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.fs.FileContext;
@@ -46,6 +41,7 @@ import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair;
import org.apache.hadoop.service.ServiceStateException;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Shell.CommandExecutor;
@@ -60,15 +56,16 @@ import
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
+import
org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerExecContext;
import
org.apache.hadoop.yarn.server.nodemanager.executor.ContainerLivenessContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReapContext;
import
org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
import
org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
import
org.apache.hadoop.yarn.server.nodemanager.executor.DeletionAsUserContext;
import
org.apache.hadoop.yarn.server.nodemanager.executor.LocalizerStartContext;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Optional;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* The {@code DefaultContainerExecuter} class offers generic container
@@ -333,7 +330,7 @@ public class DefaultContainerExecutor extends
ContainerExecutor {
builder.append("Exception from container-launch.\n")
.append("Container id: ").append(con
[hadoop] branch branch-3.3 updated: HADOOP-17111. Replace Guava Optional with Java8+ Optional. Contributed by Ahmed Hussein.
This is an automated email from the ASF dual-hosted git repository.
aajisaka pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 20df70a HADOOP-17111. Replace Guava Optional with Java8+ Optional.
Contributed by Ahmed Hussein.
20df70a is described below
commit 20df70a895d3a4c40cb1fa3fe75be543ddbc1525
Author: Akira Ajisaka
AuthorDate: Mon Jul 6 16:08:36 2020 +0900
HADOOP-17111. Replace Guava Optional with Java8+ Optional. Contributed by
Ahmed Hussein.
(cherry picked from commit 639acb6d8921127cde3174a302f2e3d71b44f052)
---
.../src/main/resources/checkstyle/checkstyle.xml | 7 ++-
.../nodemanager/DefaultContainerExecutor.java | 19 +++
.../server/nodemanager/LinuxContainerExecutor.java | 58 +++---
.../recovery/TestZKRMStateStorePerf.java | 23 -
4 files changed, 66 insertions(+), 41 deletions(-)
diff --git a/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml
b/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml
index 8f3d3f1..54a5943 100644
--- a/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml
+++ b/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml
@@ -119,7 +119,12 @@
-
+
+
+
+
+
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index c5fc481..b8f94b8 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -21,12 +21,7 @@ package org.apache.hadoop.yarn.server.nodemanager;
import static org.apache.hadoop.fs.CreateFlag.CREATE;
import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
-import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair;
-import
org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
-import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerExecContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
+import com.google.common.annotations.VisibleForTesting;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
@@ -38,7 +33,7 @@ import java.util.Arrays;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
-
+import java.util.Optional;
import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.fs.FileContext;
@@ -46,6 +41,7 @@ import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair;
import org.apache.hadoop.service.ServiceStateException;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Shell.CommandExecutor;
@@ -60,15 +56,16 @@ import
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
+import
org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerExecContext;
import
org.apache.hadoop.yarn.server.nodemanager.executor.ContainerLivenessContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReapContext;
import
org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
import
org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
import
org.apache.hadoop.yarn.server.nodemanager.executor.DeletionAsUserContext;
import
org.apache.hadoop.yarn.server.nodemanager.executor.LocalizerStartContext;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Optional;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* The {@code DefaultContainerExecuter} class offers generic container
@@ -333,7 +330,7 @@ public class DefaultContainerExecutor extends
ContainerExecutor {
builder.append
[hadoop] branch trunk updated (639acb6 -> 2f500e4)
This is an automated email from the ASF dual-hosted git repository. stevel pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git. from 639acb6 HADOOP-17111. Replace Guava Optional with Java8+ Optional. Contributed by Ahmed Hussein. add 2f500e4 HADOOP-17081. MetricsSystem doesn't start the sink adapters on restart (#2089) No new revisions were added by this update. Summary of changes: .../hadoop/metrics2/impl/MetricsSystemImpl.java | 6 +- .../hadoop/metrics2/impl/TestMetricsSystemImpl.java | 21 + 2 files changed, 26 insertions(+), 1 deletion(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.3 updated: HADOOP-17081. MetricsSystem doesn't start the sink adapters on restart (#2089)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 0789ae5 HADOOP-17081. MetricsSystem doesn't start the sink adapters
on restart (#2089)
0789ae5 is described below
commit 0789ae5b78a8ded1cea6d8db668f9fd68fa64f7a
Author: Madhusoodan Pataki
AuthorDate: Mon Jul 6 20:55:42 2020 +0530
HADOOP-17081. MetricsSystem doesn't start the sink adapters on restart
(#2089)
Contributed by Madhusoodan P
---
.../hadoop/metrics2/impl/MetricsSystemImpl.java | 6 +-
.../hadoop/metrics2/impl/TestMetricsSystemImpl.java | 21 +
2 files changed, 26 insertions(+), 1 deletion(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
index 624edc9..cf4b4a9 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
@@ -273,7 +273,11 @@ public class MetricsSystemImpl extends MetricsSystem
implements MetricsSource {
T register(final String name, final String description, final T sink) {
LOG.debug(name +", "+ description);
if (allSinks.containsKey(name)) {
- LOG.warn("Sink "+ name +" already exists!");
+ if(sinks.get(name) == null) {
+registerSink(name, description, sink);
+ } else {
+LOG.warn("Sink "+ name +" already exists!");
+ }
return sink;
}
allSinks.put(name, sink);
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
index 47a3b4c..1b40a17 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
@@ -639,4 +639,25 @@ public class TestMetricsSystemImpl {
private static String getPluginUrlsAsString() {
return "file:metrics2-test-plugin.jar";
}
+
+ @Test
+ public void testMetricSystemRestart() {
+MetricsSystemImpl ms = new MetricsSystemImpl("msRestartTestSystem");
+TestSink ts = new TestSink();
+String sinkName = "restartTestSink";
+
+try {
+ ms.start();
+ ms.register(sinkName, "", ts);
+ assertNotNull("no adapter exists for " + sinkName,
+ ms.getSinkAdapter(sinkName));
+ ms.stop();
+
+ ms.start();
+ assertNotNull("no adapter exists for " + sinkName,
+ ms.getSinkAdapter(sinkName));
+} finally {
+ ms.stop();
+}
+ }
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.2 updated: HADOOP-17081. MetricsSystem doesn't start the sink adapters on restart (#2089)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new dfb74a3 HADOOP-17081. MetricsSystem doesn't start the sink adapters
on restart (#2089)
dfb74a3 is described below
commit dfb74a3eeb43462b3c8527fbc9164caaed72c4fb
Author: Madhusoodan Pataki
AuthorDate: Mon Jul 6 20:55:42 2020 +0530
HADOOP-17081. MetricsSystem doesn't start the sink adapters on restart
(#2089)
Contributed by Madhusoodan P
---
.../hadoop/metrics2/impl/MetricsSystemImpl.java | 6 +-
.../hadoop/metrics2/impl/TestMetricsSystemImpl.java | 21 +
2 files changed, 26 insertions(+), 1 deletion(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
index 624edc9..cf4b4a9 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
@@ -273,7 +273,11 @@ public class MetricsSystemImpl extends MetricsSystem
implements MetricsSource {
T register(final String name, final String description, final T sink) {
LOG.debug(name +", "+ description);
if (allSinks.containsKey(name)) {
- LOG.warn("Sink "+ name +" already exists!");
+ if(sinks.get(name) == null) {
+registerSink(name, description, sink);
+ } else {
+LOG.warn("Sink "+ name +" already exists!");
+ }
return sink;
}
allSinks.put(name, sink);
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
index f3a2553..47520b5 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
@@ -639,4 +639,25 @@ public class TestMetricsSystemImpl {
private static String getPluginUrlsAsString() {
return "file:metrics2-test-plugin.jar";
}
+
+ @Test
+ public void testMetricSystemRestart() {
+MetricsSystemImpl ms = new MetricsSystemImpl("msRestartTestSystem");
+TestSink ts = new TestSink();
+String sinkName = "restartTestSink";
+
+try {
+ ms.start();
+ ms.register(sinkName, "", ts);
+ assertNotNull("no adapter exists for " + sinkName,
+ ms.getSinkAdapter(sinkName));
+ ms.stop();
+
+ ms.start();
+ assertNotNull("no adapter exists for " + sinkName,
+ ms.getSinkAdapter(sinkName));
+} finally {
+ ms.stop();
+}
+ }
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: HDFS-15451. Do not discard non-initial block report for provided storage. (#2119). Contributed by Shanyu Zhao.
This is an automated email from the ASF dual-hosted git repository.
hexiaoqiao pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 834372f HDFS-15451. Do not discard non-initial block report for
provided storage. (#2119). Contributed by Shanyu Zhao.
834372f is described below
commit 834372f4040f1e7a00720da5c40407f9b1423b6d
Author: Shanyu Zhao
AuthorDate: Mon Jul 6 08:43:34 2020 -0700
HDFS-15451. Do not discard non-initial block report for provided storage.
(#2119). Contributed by Shanyu Zhao.
Signed-off-by: He Xiaoqiao
---
.../hdfs/server/blockmanagement/BlockManager.java | 1 +
.../server/blockmanagement/TestBlockManager.java | 53 ++
2 files changed, 54 insertions(+)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 7f0f17e..f2cd6b9 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -2759,6 +2759,7 @@ public class BlockManager implements BlockStatsMXBean {
storageInfo = node.updateStorage(storage);
}
if (namesystem.isInStartupSafeMode()
+ && !StorageType.PROVIDED.equals(storageInfo.getStorageType())
&& storageInfo.getBlockReportCount() > 0) {
blockLog.info("BLOCK* processReport 0x{}: "
+ "discarded non-initial block report from {}"
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 11ed5ba..695377a 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -49,9 +49,11 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import
org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
+import org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.TestProvidedImpl;
import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
import org.apache.hadoop.hdfs.server.datanode.ReplicaBeingWritten;
import org.apache.hadoop.hdfs.server.namenode.CacheManager;
@@ -1052,6 +1054,57 @@ public class TestBlockManager {
}
@Test
+ public void testSafeModeWithProvidedStorageBR() throws Exception {
+DatanodeDescriptor node0 = spy(nodes.get(0));
+DatanodeStorageInfo ds0 = node0.getStorageInfos()[0];
+node0.setAlive(true);
+DatanodeDescriptor node1 = spy(nodes.get(1));
+DatanodeStorageInfo ds1 = node1.getStorageInfos()[0];
+node1.setAlive(true);
+
+String providedStorageID = DFSConfigKeys.DFS_PROVIDER_STORAGEUUID_DEFAULT;
+DatanodeStorage providedStorage = new DatanodeStorage(
+providedStorageID, DatanodeStorage.State.NORMAL, StorageType.PROVIDED);
+
+// create block manager with provided storage enabled
+Configuration conf = new HdfsConfiguration();
+conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_PROVIDED_ENABLED, true);
+conf.setClass(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_CLASS,
+TestProvidedImpl.TestFileRegionBlockAliasMap.class,
+BlockAliasMap.class);
+BlockManager bmPs = new BlockManager(fsn, false, conf);
+bmPs.setBlockPoolId("BP-12344-10.1.1.2-12344");
+
+// pretend to be in safemode
+doReturn(true).when(fsn).isInStartupSafeMode();
+
+// register new node
+DatanodeRegistration nodeReg0 =
+new DatanodeRegistration(node0, null, null, "");
+bmPs.getDatanodeManager().registerDatanode(nodeReg0);
+bmPs.getDatanodeManager().addDatanode(node0);
+DatanodeRegistration nodeReg1 =
+new DatanodeRegistration(node1, null, null, "");
+bmPs.getDatanodeManager().registerDatanode(nodeReg1);
+bmPs.getDatanodeManager().addDatanode(node1);
+
+// process reports of provided storage and disk storage
+bmPs.processReport(node0, providedStorage, BlockListAsLongs.EMPTY, null);
+bmPs.processReport(node0, new DatanodeStorage(ds0.getStorageID()),
+BlockListAsLongs.EMPTY, null);
+bmPs.processRe
[hadoop] branch branch-3.3.0 updated: Updated the index as per 3.3.0 release
This is an automated email from the ASF dual-hosted git repository.
brahma pushed a commit to branch branch-3.3.0
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3.0 by this push:
new aa96f18 Updated the index as per 3.3.0 release
aa96f18 is described below
commit aa96f1871bfd858f9bac59cf2a81ec470da649af
Author: Brahma Reddy Battula
AuthorDate: Mon Jul 6 23:24:25 2020 +0530
Updated the index as per 3.3.0 release
---
hadoop-project/src/site/markdown/index.md.vm | 237 +--
1 file changed, 39 insertions(+), 198 deletions(-)
diff --git a/hadoop-project/src/site/markdown/index.md.vm
b/hadoop-project/src/site/markdown/index.md.vm
index 438145a..78d8a47 100644
--- a/hadoop-project/src/site/markdown/index.md.vm
+++ b/hadoop-project/src/site/markdown/index.md.vm
@@ -16,10 +16,7 @@ Apache Hadoop ${project.version}
Apache Hadoop ${project.version} incorporates a number of significant
-enhancements over the previous major release line (hadoop-2.x).
-
-This release is generally available (GA), meaning that it represents a point of
-API stability and quality that we consider production-ready.
+enhancements over the previous major release line (hadoop-3.2).
Overview
@@ -27,224 +24,68 @@ Overview
Users are encouraged to read the full set of release notes.
This page provides an overview of the major changes.
-Minimum required Java version increased from Java 7 to Java 8
---
+ARM Support
+
+This is the first release to support ARM architectures.
-All Hadoop JARs are now compiled targeting a runtime version of Java 8.
-Users still using Java 7 or below must upgrade to Java 8.
+Upgrade protobuf from 2.5.0 to something newer
+-
+Protobuf upgraded to 3.7.1 as protobuf-2.5.0 reached EOL.
-Support for erasure coding in HDFS
+Java 11 runtime support
--
-Erasure coding is a method for durably storing data with significant space
-savings compared to replication. Standard encodings like Reed-Solomon (10,4)
-have a 1.4x space overhead, compared to the 3x overhead of standard HDFS
-replication.
-
-Since erasure coding imposes additional overhead during reconstruction
-and performs mostly remote reads, it has traditionally been used for
-storing colder, less frequently accessed data. Users should consider
-the network and CPU overheads of erasure coding when deploying this
-feature.
-
-More details are available in the
-[HDFS Erasure Coding](./hadoop-project-dist/hadoop-hdfs/HDFSErasureCoding.html)
-documentation.
-
-YARN Timeline Service v.2
-
-We are introducing an early preview (alpha 2) of a major revision of YARN
-Timeline Service: v.2. YARN Timeline Service v.2 addresses two major
-challenges: improving scalability and reliability of Timeline Service, and
-enhancing usability by introducing flows and aggregation.
-
-YARN Timeline Service v.2 alpha 2 is provided so that users and developers
-can test it and provide feedback and suggestions for making it a ready
-replacement for Timeline Service v.1.x. It should be used only in a test
-capacity.
-
-More details are available in the
-[YARN Timeline Service
v.2](./hadoop-yarn/hadoop-yarn-site/TimelineServiceV2.html)
-documentation.
-
-Shell script rewrite
+Java 11 runtime support is completed.
-The Hadoop shell scripts have been rewritten to fix many long-standing
-bugs and include some new features. While an eye has been kept towards
-compatibility, some changes may break existing installations.
+Support impersonation for AuthenticationFilter
+-
-Incompatible changes are documented in the release notes, with related
-discussion on [HADOOP-9902](https://issues.apache.org/jira/browse/HADOOP-9902).
+External services or YARN service may need to call into WebHDFS or YARN REST
API on behave of the user using web
+protocols. It would be good to support impersonation mechanism in
AuthenticationFilter or similar extensions.
-More details are available in the
-[Unix Shell Guide](./hadoop-project-dist/hadoop-common/UnixShellGuide.html)
-documentation. Power users will also be pleased by the
-[Unix Shell API](./hadoop-project-dist/hadoop-common/UnixShellAPI.html)
-documentation, which describes much of the new functionality, particularly
-related to extensibility.
-Shaded client jars
+s3A Enhancements
--
+Lots of enhancements to the S3A code including Delegation Token support,
better handling of 404 caching,
+ S3guard performance, resilience improvements
-The `hadoop-client` Maven artifact available in 2.x releases pulls
-Hadoop's transitive dependencies onto a Hadoop application's classpath.
-This can be problematic if the versions of these transitive dependencies
-conflict with the versions used by the application.
-
-
[hadoop] annotated tag release-3.3.0-RC0 created (now 970fe5b)
This is an automated email from the ASF dual-hosted git repository. brahma pushed a change to annotated tag release-3.3.0-RC0 in repository https://gitbox.apache.org/repos/asf/hadoop.git. at 970fe5b (tag) tagging aa96f1871bfd858f9bac59cf2a81ec470da649af (commit) replaces remove-ozone by Brahma Reddy Battula on Tue Jul 7 03:01:07 2020 +0530 - Log - Release candidate - hadoop-3.3.0-RC0 -BEGIN PGP SIGNATURE- Version: GnuPG v1 iQIcBAABAgAGBQJfA5gbAAoJEDhtgO+B50aaoS0QAI9J0dls4aOXc6N0rYinnG7e UXZuAAQTXRVtwLSnS6BG3jkU2ocgVSo5GJCw3+MVpcIcR3/BpoY7taLZWxshXI/I S0KpANwcz8eiHR8jFsI1C+3HMKDFdE05bZqOTMLBHes5DyWHAdTSJfnpdKMtumH5 KxVrcYCyHvAEAG7kr/3dTg/BedPU81mdmFE9tihQGBC2eAUEIojUVaOzXFCJK1oV oacaaWYha3ryzL9E+VJCOMqZEYbzYsFV4XAhxvGGZMvuKx6MnKDWdf1xYsFi+USI 0uJhQsrAk4NjdM5Ve4nVo0r2LnCuEQ5HkCFADCc9LlRPYkkwgn3xjtGxjR8S6iiz HF+ZrlxYNSnueNcPv7/bgIVbjLLb08rvOjZ2nwtEiA+G6Zr0o5HSA0/Yg5zwjEmw tLpnyoRSSQXqn+RGyh2BCuYZ+3TOakZF3muscyMDy+mCibpb6pW8EZsZeg+0Bzu4 IIZbcErGJDgJGrInuUFjj0b0WbJ4vDVVA6a6ZCeCC0ySladkeXeztwCWNRNMfWxj Li8xPf5O39h3jICEvKTE6ePTVMpKwBakiasif1qi/HD810Avhq8PU2IemrmiCXhz s2sQ2tQNrSL4w7WjV83dJkR5jmBga/Lsk9B2FSXgO5ZLfo+5q+vn0mnVhYPgP8/O aSkbMa/eBRo9UsZ1d/iJ =KHBg -END PGP SIGNATURE- --- No new revisions were added by this update. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: HDFS-15417. RBF: Get the datanode report from cache for federation WebHDFS operations (#2080)
This is an automated email from the ASF dual-hosted git repository.
sunchao pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new e820baa6 HDFS-15417. RBF: Get the datanode report from cache for
federation WebHDFS operations (#2080)
e820baa6 is described below
commit e820baa6e6f7e850ba62cbf150d760bd0ea6d0e0
Author: Ye Ni <141253+nick...@users.noreply.github.com>
AuthorDate: Mon Jul 6 16:17:09 2020 -0700
HDFS-15417. RBF: Get the datanode report from cache for federation WebHDFS
operations (#2080)
---
.../server/federation/router/RouterRpcServer.java | 123 -
.../federation/router/RouterWebHdfsMethods.java| 9 +-
.../server/federation/router/TestRouterRpc.java| 70 +++-
3 files changed, 191 insertions(+), 11 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
index 4f1310b..5905a1d 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
@@ -26,6 +26,8 @@ import static
org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_
import static
org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_READER_COUNT_KEY;
import static
org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_READER_QUEUE_SIZE_DEFAULT;
import static
org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_READER_QUEUE_SIZE_KEY;
+import static
org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DN_REPORT_CACHE_EXPIRE;
+import static
org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DN_REPORT_CACHE_EXPIRE_MS_DEFAULT;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -41,7 +43,19 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.CryptoProtocolVersion;
import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries;
@@ -219,6 +233,9 @@ public class RouterRpcServer extends AbstractService
implements ClientProtocol,
private static final ThreadLocal CUR_USER =
new ThreadLocal<>();
+ /** DN type -> full DN report. */
+ private final LoadingCache dnCache;
+
/**
* Construct a router RPC server.
*
@@ -361,6 +378,23 @@ public class RouterRpcServer extends AbstractService
implements ClientProtocol,
this.nnProto = new RouterNamenodeProtocol(this);
this.clientProto = new RouterClientProtocol(conf, this);
this.routerProto = new RouterUserProtocol(this);
+
+long dnCacheExpire = conf.getTimeDuration(
+DN_REPORT_CACHE_EXPIRE,
+DN_REPORT_CACHE_EXPIRE_MS_DEFAULT, TimeUnit.MILLISECONDS);
+this.dnCache = CacheBuilder.newBuilder()
+.build(new DatanodeReportCacheLoader());
+
+// Actively refresh the dn cache in a configured interval
+Executors
+.newSingleThreadScheduledExecutor()
+.scheduleWithFixedDelay(() -> this.dnCache
+.asMap()
+.keySet()
+.parallelStream()
+.forEach((key) -> this.dnCache.refresh(key)),
+0,
+dnCacheExpire, TimeUnit.MILLISECONDS);
}
@Override
@@ -869,6 +903,50 @@ public class RouterRpcServer extends AbstractService
implements ClientProtocol,
}
/**
+ * Get the datanode report from cache.
+ *
+ * @param type Type of the datanode.
+ * @return List of datanodes.
+ * @throws IOException If it cannot get the report.
+ */
+ DatanodeInfo[] getCachedDatanodeReport(DatanodeReportType type)
+ throws IOException {
+try {
+ DatanodeInfo[] dns = this.dnCache.get(type);
+ if (dns == null) {
+LOG.debug("Get null DN report from cache");
+dns = getCachedDatanodeReportImpl(type);
+this.dnCache.put(type, dns);
+ }
+ return dns;
+} catch (ExecutionException e) {
+ LOG.error("Cannot
[hadoop] branch trunk updated: HDFS-15449. Optionally ignore port number in mount-table name when picking from initialized uri. Contributed by Uma Maheswara Rao G.
This is an automated email from the ASF dual-hosted git repository.
umamahesh pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new dc0626b HDFS-15449. Optionally ignore port number in mount-table name
when picking from initialized uri. Contributed by Uma Maheswara Rao G.
dc0626b is described below
commit dc0626b5f2f2ba0bd3919650ea231cedd424f77a
Author: Uma Maheswara Rao G
AuthorDate: Mon Jul 6 18:50:03 2020 -0700
HDFS-15449. Optionally ignore port number in mount-table name when picking
from initialized uri. Contributed by Uma Maheswara Rao G.
---
.../org/apache/hadoop/fs/viewfs/Constants.java | 13 +++
.../apache/hadoop/fs/viewfs/ViewFileSystem.java| 10 -
.../fs/viewfs/ViewFileSystemOverloadScheme.java| 13 ++-
.../src/site/markdown/ViewFsOverloadScheme.md | 8 +++-
...SystemOverloadSchemeHdfsFileSystemContract.java | 4 ++
...ViewFileSystemOverloadSchemeWithHdfsScheme.java | 45 +-
...stViewFileSystemOverloadSchemeWithDFSAdmin.java | 17
...ViewFileSystemOverloadSchemeWithFSCommands.java | 2 +-
8 files changed, 97 insertions(+), 15 deletions(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
index 28ebf73..492cb87 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
@@ -104,4 +104,17 @@ public interface Constants {
"fs.viewfs.mount.links.as.symlinks";
boolean CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS_DEFAULT = true;
+
+ /**
+ * When initializing the viewfs, authority will be used as the mount table
+ * name to find the mount link configurations. To make the mount table name
+ * unique, we may want to ignore port if initialized uri authority contains
+ * port number. By default, we will consider port number also in
+ * ViewFileSystem(This default value false, because to support existing
+ * deployments continue with the current behavior).
+ */
+ String CONFIG_VIEWFS_IGNORE_PORT_IN_MOUNT_TABLE_NAME =
+ "fs.viewfs.ignore.port.in.mount.table.name";
+
+ boolean CONFIG_VIEWFS_IGNORE_PORT_IN_MOUNT_TABLE_NAME_DEFAULT = false;
}
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index cb36965..0beeda2 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.fs.viewfs;
import static
org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs;
import static
org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE;
import static
org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE_DEFAULT;
+import static
org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_IGNORE_PORT_IN_MOUNT_TABLE_NAME;
+import static
org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_IGNORE_PORT_IN_MOUNT_TABLE_NAME_DEFAULT;
import static
org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS;
import static
org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS_DEFAULT;
import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555;
@@ -274,9 +276,15 @@ public class ViewFileSystem extends FileSystem {
final InnerCache innerCache = new InnerCache(fsGetter);
// Now build client side view (i.e. client side mount table) from config.
final String authority = theUri.getAuthority();
+String tableName = authority;
+if (theUri.getPort() != -1 && config
+.getBoolean(CONFIG_VIEWFS_IGNORE_PORT_IN_MOUNT_TABLE_NAME,
+CONFIG_VIEWFS_IGNORE_PORT_IN_MOUNT_TABLE_NAME_DEFAULT)) {
+ tableName = theUri.getHost();
+}
try {
myUri = new URI(getScheme(), authority, "/", null, null);
- fsState = new InodeTree(conf, authority) {
+ fsState = new InodeTree(conf, tableName) {
@Override
protected FileSystem getTargetFileSystem(final URI uri)
throws URISyntaxException, IOException {
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemOverloadScheme.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemOverloadScheme.java
index 672022b..2f3359d 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemOverloadScheme.java
+++
b/hadoop-commo
[hadoop] branch trunk updated (dc0626b -> f77bbc2)
This is an automated email from the ASF dual-hosted git repository. inigoiri pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git. from dc0626b HDFS-15449. Optionally ignore port number in mount-table name when picking from initialized uri. Contributed by Uma Maheswara Rao G. add f77bbc2 HDFS-15312. Apply umask when creating directory by WebHDFS (#2096) No new revisions were added by this update. Summary of changes: .../src/main/webapps/router/explorer.js| 29 ++ .../hadoop-hdfs/src/main/webapps/hdfs/explorer.js | 29 ++ 2 files changed, 58 insertions(+) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.1 updated: HDFS-15451. Do not discard non-initial block report for provided storage. (#2119). Contributed by Shanyu Zhao.
This is an automated email from the ASF dual-hosted git repository.
hexiaoqiao pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new d787b93 HDFS-15451. Do not discard non-initial block report for
provided storage. (#2119). Contributed by Shanyu Zhao.
d787b93 is described below
commit d787b9373a3302c704da4e9bc6e8549b523c9bed
Author: Shanyu Zhao
AuthorDate: Mon Jul 6 08:43:34 2020 -0700
HDFS-15451. Do not discard non-initial block report for provided storage.
(#2119). Contributed by Shanyu Zhao.
Signed-off-by: He Xiaoqiao
---
.../hdfs/server/blockmanagement/BlockManager.java | 1 +
.../server/blockmanagement/TestBlockManager.java | 53 ++
2 files changed, 54 insertions(+)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 17d5603..b80e58f 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -2689,6 +2689,7 @@ public class BlockManager implements BlockStatsMXBean {
storageInfo = node.updateStorage(storage);
}
if (namesystem.isInStartupSafeMode()
+ && !StorageType.PROVIDED.equals(storageInfo.getStorageType())
&& storageInfo.getBlockReportCount() > 0) {
blockLog.info("BLOCK* processReport 0x{}: "
+ "discarded non-initial block report from {}"
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 623ab1a..6adbb95 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -47,9 +47,11 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import
org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
+import org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.TestProvidedImpl;
import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
import org.apache.hadoop.hdfs.server.datanode.ReplicaBeingWritten;
import org.apache.hadoop.hdfs.server.namenode.CacheManager;
@@ -1003,6 +1005,57 @@ public class TestBlockManager {
}
@Test
+ public void testSafeModeWithProvidedStorageBR() throws Exception {
+DatanodeDescriptor node0 = spy(nodes.get(0));
+DatanodeStorageInfo ds0 = node0.getStorageInfos()[0];
+node0.setAlive(true);
+DatanodeDescriptor node1 = spy(nodes.get(1));
+DatanodeStorageInfo ds1 = node1.getStorageInfos()[0];
+node1.setAlive(true);
+
+String providedStorageID = DFSConfigKeys.DFS_PROVIDER_STORAGEUUID_DEFAULT;
+DatanodeStorage providedStorage = new DatanodeStorage(
+providedStorageID, DatanodeStorage.State.NORMAL, StorageType.PROVIDED);
+
+// create block manager with provided storage enabled
+Configuration conf = new HdfsConfiguration();
+conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_PROVIDED_ENABLED, true);
+conf.setClass(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_CLASS,
+TestProvidedImpl.TestFileRegionBlockAliasMap.class,
+BlockAliasMap.class);
+BlockManager bmPs = new BlockManager(fsn, false, conf);
+bmPs.setBlockPoolId("BP-12344-10.1.1.2-12344");
+
+// pretend to be in safemode
+doReturn(true).when(fsn).isInStartupSafeMode();
+
+// register new node
+DatanodeRegistration nodeReg0 =
+new DatanodeRegistration(node0, null, null, "");
+bmPs.getDatanodeManager().registerDatanode(nodeReg0);
+bmPs.getDatanodeManager().addDatanode(node0);
+DatanodeRegistration nodeReg1 =
+new DatanodeRegistration(node1, null, null, "");
+bmPs.getDatanodeManager().registerDatanode(nodeReg1);
+bmPs.getDatanodeManager().addDatanode(node1);
+
+// process reports of provided storage and disk storage
+bmPs.processReport(node0, providedStorage, BlockListAsLongs.EMPTY, null);
+bmPs.processReport(node0, new DatanodeStorage(ds0.getStorageID()),
+BlockListAsLongs.EMPTY, null);
+bmPs
[hadoop] branch branch-3.2 updated: HDFS-15451. Do not discard non-initial block report for provided storage. (#2119). Contributed by Shanyu Zhao.
This is an automated email from the ASF dual-hosted git repository.
hexiaoqiao pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 8d40288 HDFS-15451. Do not discard non-initial block report for
provided storage. (#2119). Contributed by Shanyu Zhao.
8d40288 is described below
commit 8d4028864d0492e8f64f94db2339fbd55f97083a
Author: Shanyu Zhao
AuthorDate: Mon Jul 6 08:43:34 2020 -0700
HDFS-15451. Do not discard non-initial block report for provided storage.
(#2119). Contributed by Shanyu Zhao.
Signed-off-by: He Xiaoqiao
---
.../hdfs/server/blockmanagement/BlockManager.java | 1 +
.../server/blockmanagement/TestBlockManager.java | 53 ++
2 files changed, 54 insertions(+)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 3185f1d..4486e47 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -2702,6 +2702,7 @@ public class BlockManager implements BlockStatsMXBean {
storageInfo = node.updateStorage(storage);
}
if (namesystem.isInStartupSafeMode()
+ && !StorageType.PROVIDED.equals(storageInfo.getStorageType())
&& storageInfo.getBlockReportCount() > 0) {
blockLog.info("BLOCK* processReport 0x{}: "
+ "discarded non-initial block report from {}"
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index d393eaa..fea80e5 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -46,9 +46,11 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import
org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
+import org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.TestProvidedImpl;
import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
import org.apache.hadoop.hdfs.server.datanode.ReplicaBeingWritten;
import org.apache.hadoop.hdfs.server.namenode.CacheManager;
@@ -1002,6 +1004,57 @@ public class TestBlockManager {
}
@Test
+ public void testSafeModeWithProvidedStorageBR() throws Exception {
+DatanodeDescriptor node0 = spy(nodes.get(0));
+DatanodeStorageInfo ds0 = node0.getStorageInfos()[0];
+node0.setAlive(true);
+DatanodeDescriptor node1 = spy(nodes.get(1));
+DatanodeStorageInfo ds1 = node1.getStorageInfos()[0];
+node1.setAlive(true);
+
+String providedStorageID = DFSConfigKeys.DFS_PROVIDER_STORAGEUUID_DEFAULT;
+DatanodeStorage providedStorage = new DatanodeStorage(
+providedStorageID, DatanodeStorage.State.NORMAL, StorageType.PROVIDED);
+
+// create block manager with provided storage enabled
+Configuration conf = new HdfsConfiguration();
+conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_PROVIDED_ENABLED, true);
+conf.setClass(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_CLASS,
+TestProvidedImpl.TestFileRegionBlockAliasMap.class,
+BlockAliasMap.class);
+BlockManager bmPs = new BlockManager(fsn, false, conf);
+bmPs.setBlockPoolId("BP-12344-10.1.1.2-12344");
+
+// pretend to be in safemode
+doReturn(true).when(fsn).isInStartupSafeMode();
+
+// register new node
+DatanodeRegistration nodeReg0 =
+new DatanodeRegistration(node0, null, null, "");
+bmPs.getDatanodeManager().registerDatanode(nodeReg0);
+bmPs.getDatanodeManager().addDatanode(node0);
+DatanodeRegistration nodeReg1 =
+new DatanodeRegistration(node1, null, null, "");
+bmPs.getDatanodeManager().registerDatanode(nodeReg1);
+bmPs.getDatanodeManager().addDatanode(node1);
+
+// process reports of provided storage and disk storage
+bmPs.processReport(node0, providedStorage, BlockListAsLongs.EMPTY, null);
+bmPs.processReport(node0, new DatanodeStorage(ds0.getStorageID()),
+BlockListAsLongs.EMPTY, null);
+bmPs
[hadoop] branch branch-3.3 updated: HDFS-15451. Do not discard non-initial block report for provided storage. (#2119). Contributed by Shanyu Zhao.
This is an automated email from the ASF dual-hosted git repository.
hexiaoqiao pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 10c9df1 HDFS-15451. Do not discard non-initial block report for
provided storage. (#2119). Contributed by Shanyu Zhao.
10c9df1 is described below
commit 10c9df1d0a75b4f8d2736bf09cce67ff22b56ded
Author: Shanyu Zhao
AuthorDate: Mon Jul 6 08:43:34 2020 -0700
HDFS-15451. Do not discard non-initial block report for provided storage.
(#2119). Contributed by Shanyu Zhao.
Signed-off-by: He Xiaoqiao
---
.../hdfs/server/blockmanagement/BlockManager.java | 1 +
.../server/blockmanagement/TestBlockManager.java | 53 ++
2 files changed, 54 insertions(+)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index ad61c71..e3da9a1 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -2745,6 +2745,7 @@ public class BlockManager implements BlockStatsMXBean {
storageInfo = node.updateStorage(storage);
}
if (namesystem.isInStartupSafeMode()
+ && !StorageType.PROVIDED.equals(storageInfo.getStorageType())
&& storageInfo.getBlockReportCount() > 0) {
blockLog.info("BLOCK* processReport 0x{}: "
+ "discarded non-initial block report from {}"
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 11ed5ba..695377a 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -49,9 +49,11 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import
org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
+import org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.TestProvidedImpl;
import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
import org.apache.hadoop.hdfs.server.datanode.ReplicaBeingWritten;
import org.apache.hadoop.hdfs.server.namenode.CacheManager;
@@ -1052,6 +1054,57 @@ public class TestBlockManager {
}
@Test
+ public void testSafeModeWithProvidedStorageBR() throws Exception {
+DatanodeDescriptor node0 = spy(nodes.get(0));
+DatanodeStorageInfo ds0 = node0.getStorageInfos()[0];
+node0.setAlive(true);
+DatanodeDescriptor node1 = spy(nodes.get(1));
+DatanodeStorageInfo ds1 = node1.getStorageInfos()[0];
+node1.setAlive(true);
+
+String providedStorageID = DFSConfigKeys.DFS_PROVIDER_STORAGEUUID_DEFAULT;
+DatanodeStorage providedStorage = new DatanodeStorage(
+providedStorageID, DatanodeStorage.State.NORMAL, StorageType.PROVIDED);
+
+// create block manager with provided storage enabled
+Configuration conf = new HdfsConfiguration();
+conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_PROVIDED_ENABLED, true);
+conf.setClass(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_CLASS,
+TestProvidedImpl.TestFileRegionBlockAliasMap.class,
+BlockAliasMap.class);
+BlockManager bmPs = new BlockManager(fsn, false, conf);
+bmPs.setBlockPoolId("BP-12344-10.1.1.2-12344");
+
+// pretend to be in safemode
+doReturn(true).when(fsn).isInStartupSafeMode();
+
+// register new node
+DatanodeRegistration nodeReg0 =
+new DatanodeRegistration(node0, null, null, "");
+bmPs.getDatanodeManager().registerDatanode(nodeReg0);
+bmPs.getDatanodeManager().addDatanode(node0);
+DatanodeRegistration nodeReg1 =
+new DatanodeRegistration(node1, null, null, "");
+bmPs.getDatanodeManager().registerDatanode(nodeReg1);
+bmPs.getDatanodeManager().addDatanode(node1);
+
+// process reports of provided storage and disk storage
+bmPs.processReport(node0, providedStorage, BlockListAsLongs.EMPTY, null);
+bmPs.processReport(node0, new DatanodeStorage(ds0.getStorageID()),
+BlockListAsLongs.EMPTY, null);
+bmPs
