[hadoop] branch trunk updated: HDFS-16378. Add datanode address to BlockReportLeaseManager logs (#3786). Contributed by tomscut.
This is an automated email from the ASF dual-hosted git repository.
hexiaoqiao pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 59c6508 HDFS-16378. Add datanode address to BlockReportLeaseManager
logs (#3786). Contributed by tomscut.
59c6508 is described below
commit 59c650802d26d501e622f77ca7c5b66f4c2a8479
Author: litao
AuthorDate: Wed Dec 15 11:16:32 2021 +0800
HDFS-16378. Add datanode address to BlockReportLeaseManager logs (#3786).
Contributed by tomscut.
Signed-off-by: He Xiaoqiao
---
.../blockmanagement/BlockReportLeaseManager.java | 58 +++---
1 file changed, 29 insertions(+), 29 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReportLeaseManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReportLeaseManager.java
index f45daac..2e7e78d 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReportLeaseManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReportLeaseManager.java
@@ -190,8 +190,8 @@ class BlockReportLeaseManager {
private synchronized NodeData registerNode(DatanodeDescriptor dn) {
if (nodes.containsKey(dn.getDatanodeUuid())) {
- LOG.info("Can't register DN {} because it is already registered.",
- dn.getDatanodeUuid());
+ LOG.info("Can't register DN {} ({}) because it is already registered.",
+ dn.getDatanodeUuid(), dn.getXferAddr());
return null;
}
NodeData node = new NodeData(dn.getDatanodeUuid());
@@ -213,8 +213,8 @@ class BlockReportLeaseManager {
public synchronized void unregister(DatanodeDescriptor dn) {
NodeData node = nodes.remove(dn.getDatanodeUuid());
if (node == null) {
- LOG.info("Can't unregister DN {} because it is not currently " +
- "registered.", dn.getDatanodeUuid());
+ LOG.info("Can't unregister DN {} ({}) because it is not currently " +
+ "registered.", dn.getDatanodeUuid(), dn.getXferAddr());
return;
}
remove(node);
@@ -224,7 +224,7 @@ class BlockReportLeaseManager {
NodeData node = nodes.get(dn.getDatanodeUuid());
if (node == null) {
LOG.warn("DN {} ({}) requested a lease even though it wasn't yet " +
- "registered. Registering now.", dn.getDatanodeUuid(),
+ "registered. Registering now.", dn.getDatanodeUuid(),
dn.getXferAddr());
node = registerNode(dn);
}
@@ -232,9 +232,9 @@ class BlockReportLeaseManager {
// The DataNode wants a new lease, even though it already has one.
// This can happen if the DataNode is restarted in between requesting
// a lease and using it.
- LOG.debug("Removing existing BR lease 0x{} for DN {} in order to " +
+ LOG.debug("Removing existing BR lease 0x{} for DN {} ({}) in order to " +
"issue a new one.", Long.toHexString(node.leaseId),
- dn.getDatanodeUuid());
+ dn.getDatanodeUuid(), dn.getXferAddr());
}
remove(node);
long monotonicNowMs = Time.monotonicNow();
@@ -248,9 +248,9 @@ class BlockReportLeaseManager {
allLeases.append(prefix).append(cur.datanodeUuid);
prefix = ", ";
}
-LOG.debug("Can't create a new BR lease for DN {}, because " +
- "numPending equals maxPending at {}. Current leases: {}",
- dn.getDatanodeUuid(), numPending, allLeases.toString());
+LOG.debug("Can't create a new BR lease for DN {} ({}), because " +
+ "numPending equals maxPending at {}. Current leases: {}",
+ dn.getDatanodeUuid(), dn.getXferAddr(), numPending, allLeases);
}
return 0;
}
@@ -259,8 +259,8 @@ class BlockReportLeaseManager {
node.leaseTimeMs = monotonicNowMs;
pendingHead.addToEnd(node);
if (LOG.isDebugEnabled()) {
- LOG.debug("Created a new BR lease 0x{} for DN {}. numPending = {}",
- Long.toHexString(node.leaseId), dn.getDatanodeUuid(), numPending);
+ LOG.debug("Created a new BR lease 0x{} for DN {} ({}). numPending = {}",
+ Long.toHexString(node.leaseId), dn.getDatanodeUuid(),
dn.getXferAddr(), numPending);
}
return node.leaseId;
}
@@ -293,36 +293,36 @@ class BlockReportLeaseManager {
public synchronized boolean checkLease(DatanodeDescriptor dn,
long monotonicNowMs, long id) {
if (id == 0) {
- LOG.debug("Datanode {} is using BR lease id 0x0 to bypass " +
- "rate-limiting.", dn.getDatanodeUuid());
+ LOG.debug("Datanode {} ({}) is using BR lease id 0x0 to bypass " +
+ "rate-limiting.", dn.getDatanodeUuid(), dn.getXferAddr());
[hadoop] branch branch-3.3.2 updated: HADOOP-18048. [branch-3.3] Dockerfile_aarch64 build fails with fatal error: Python.h: No such file or directory (#3801)
This is an automated email from the ASF dual-hosted git repository. sunchao pushed a commit to branch branch-3.3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.3.2 by this push: new d4794dd HADOOP-18048. [branch-3.3] Dockerfile_aarch64 build fails with fatal error: Python.h: No such file or directory (#3801) d4794dd is described below commit d4794dd3b2ba365a9d95ad6aafcf43a1ea40f777 Author: Siyao Meng <50227127+smen...@users.noreply.github.com> AuthorDate: Tue Dec 14 17:56:58 2021 -0800 HADOOP-18048. [branch-3.3] Dockerfile_aarch64 build fails with fatal error: Python.h: No such file or directory (#3801) --- dev-support/docker/Dockerfile_aarch64 | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-support/docker/Dockerfile_aarch64 b/dev-support/docker/Dockerfile_aarch64 index 528e78c..80e813b 100644 --- a/dev-support/docker/Dockerfile_aarch64 +++ b/dev-support/docker/Dockerfile_aarch64 @@ -74,6 +74,7 @@ RUN apt-get -q update \ pkg-config \ python2.7 \ python3 \ +python3-dev \ python3-pip \ python3-pkg-resources \ python3-setuptools \ - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.3 updated: HADOOP-18048. [branch-3.3] Dockerfile_aarch64 build fails with fatal error: Python.h: No such file or directory (#3801)
This is an automated email from the ASF dual-hosted git repository. sunchao pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.3 by this push: new 0cab549 HADOOP-18048. [branch-3.3] Dockerfile_aarch64 build fails with fatal error: Python.h: No such file or directory (#3801) 0cab549 is described below commit 0cab54909a1606693d336106b966640374525981 Author: Siyao Meng <50227127+smen...@users.noreply.github.com> AuthorDate: Tue Dec 14 17:56:58 2021 -0800 HADOOP-18048. [branch-3.3] Dockerfile_aarch64 build fails with fatal error: Python.h: No such file or directory (#3801) --- dev-support/docker/Dockerfile_aarch64 | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-support/docker/Dockerfile_aarch64 b/dev-support/docker/Dockerfile_aarch64 index 528e78c..80e813b 100644 --- a/dev-support/docker/Dockerfile_aarch64 +++ b/dev-support/docker/Dockerfile_aarch64 @@ -74,6 +74,7 @@ RUN apt-get -q update \ pkg-config \ python2.7 \ python3 \ +python3-dev \ python3-pip \ python3-pkg-resources \ python3-setuptools \ - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: Clean up checkstyle warnings from YARN-11024/10907/10929. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 843f66f Clean up checkstyle warnings from YARN-11024/10907/10929.
Contributed by Benjamin Teke
843f66f is described below
commit 843f66f4dc012dff402bfebc183d46673cd47419
Author: Szilard Nemeth
AuthorDate: Tue Dec 14 22:00:43 2021 +0100
Clean up checkstyle warnings from YARN-11024/10907/10929. Contributed by
Benjamin Teke
---
.../scheduler/capacity/AbstractCSQueue.java| 3 +-
.../scheduler/capacity/AbstractLeafQueue.java | 38 +-
.../GuaranteedOrZeroCapacityOverTimePolicy.java| 1 -
.../scheduler/capacity/TestApplicationLimits.java | 35 +---
.../capacity/TestApplicationLimitsByPartition.java | 1 -
.../scheduler/capacity/TestReservations.java | 1 -
6 files changed, 33 insertions(+), 46 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 3a0e2ae..809a860 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -145,7 +145,8 @@ public abstract class AbstractCSQueue implements CSQueue {
CSQueueMetrics metrics = old != null ?
(CSQueueMetrics) old.getMetrics() :
CSQueueMetrics.forQueue(getQueuePath(), parent,
-queueContext.getConfiguration().getEnableUserMetrics(),
queueContext.getConfiguration());
+queueContext.getConfiguration().getEnableUserMetrics(),
+queueContext.getConfiguration());
this.usageTracker = new CSQueueUsageTracker(metrics);
this.queueCapacities = new QueueCapacities(parent == null);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java
index 8b31241..e194800 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java
@@ -453,8 +453,7 @@ public class AbstractLeafQueue extends AbstractCSQueue {
}
@Override
- public List
- getQueueUserAclInfo(UserGroupInformation user) {
+ public List getQueueUserAclInfo(UserGroupInformation user)
{
readLock.lock();
try {
QueueUserACLInfo userAclInfo = recordFactory.newRecordInstance(
@@ -527,8 +526,8 @@ public class AbstractLeafQueue extends AbstractCSQueue {
}
@Override
- public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
throws
- IOException {
+ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
+ throws IOException {
writeLock.lock();
try {
@@ -634,7 +633,8 @@ public class AbstractLeafQueue extends AbstractCSQueue {
// Check submission limits for queues
//TODO recalculate max applications because they can depend on capacity
- if (getNumApplications() >= getMaxApplications() && !(this instanceof
AutoCreatedLeafQueue)) {
+ if (getNumApplications() >= getMaxApplications() &&
+ !(this instanceof AutoCreatedLeafQueue)) {
String msg =
"Queue " + getQueuePath() + " already has " + getNumApplications()
+ " applications,"
@@ -646,7 +646,8 @@ public class AbstractLeafQueue extends AbstractCSQueue {
// Check submission limits for the user on this queue
User user = usersManager.getUserAndAddIfAbsent(userName);
//TODO recalculate max applications because they can depend on capacity
- if (user.getTotalApplications() >= getMaxApplicationsPerUser() &&
!(this instanceof AutoCreatedLeafQueue)) {
+ if (user.getTotalApplications() >= getMaxApplicationsPerUser() &&
+ !(this instanceof AutoCreatedLeafQueue)) {
[hadoop] branch branch-3.2 updated: HADOOP-17534. Update Jackson to 2.10.5 and Jackson databind to 2.10.5.1 (#2708)
This is an automated email from the ASF dual-hosted git repository. aajisaka pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.2 by this push: new 07d3d5e HADOOP-17534. Update Jackson to 2.10.5 and Jackson databind to 2.10.5.1 (#2708) 07d3d5e is described below commit 07d3d5ea0b1e56c2f6027536d4c4667dbb9c8512 Author: Akira Ajisaka AuthorDate: Mon Feb 22 17:33:02 2021 +0900 HADOOP-17534. Update Jackson to 2.10.5 and Jackson databind to 2.10.5.1 (#2708) Reviewed-by: Wei-Chiu Chuang (cherry picked from commit e119063a078f06523158dba3b9e16ca04768e920) --- hadoop-project/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 4d4e166..636b9b1 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -71,8 +71,8 @@ 1.9.13 -2.10.3 -2.10.3 +2.10.5 +2.10.5.1 4.5.13 - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10929. Do not use a separate config in legacy CS AQC. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 364d38f YARN-10929. Do not use a separate config in legacy CS AQC.
Contributed by Benjamin Teke
364d38f is described below
commit 364d38f00d75e06099ef030030fca7ec90a8a817
Author: Szilard Nemeth
AuthorDate: Tue Dec 14 13:41:22 2021 +0100
YARN-10929. Do not use a separate config in legacy CS AQC. Contributed by
Benjamin Teke
---
.../scheduler/capacity/AbstractCSQueue.java| 44 ++
.../scheduler/capacity/AbstractLeafQueue.java | 69 +-
.../capacity/AbstractManagedParentQueue.java | 17 +-
.../scheduler/capacity/AutoCreatedLeafQueue.java | 10 ++--
.../capacity/CSQueuePreemptionSettings.java| 19 +++---
.../capacity/CapacitySchedulerQueueContext.java| 4 ++
.../scheduler/capacity/LeafQueue.java | 2 +-
.../scheduler/capacity/ManagedParentQueue.java | 28 +++--
.../scheduler/capacity/ParentQueue.java| 10 ++--
.../scheduler/capacity/PlanQueue.java | 2 +-
.../capacity/QueueAllocationSettings.java | 11 +---
.../scheduler/capacity/ReservationQueue.java | 6 +-
.../capacity/TestCapacitySchedulerPerf.java| 2 +-
13 files changed, 84 insertions(+), 140 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 5040b02..3a0e2ae 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -173,10 +173,9 @@ public abstract class AbstractCSQueue implements CSQueue {
return new QueuePath(parent.getQueuePath(), queueName);
}
- protected void setupConfigurableCapacities(
- CapacitySchedulerConfiguration configuration) {
+ protected void setupConfigurableCapacities() {
CSQueueUtils.loadCapacitiesByLabelsFromConf(queuePath, queueCapacities,
-configuration, this.queueNodeLabelsSettings.getConfiguredNodeLabels());
+queueContext.getConfiguration(),
this.queueNodeLabelsSettings.getConfiguredNodeLabels());
}
@Override
@@ -329,14 +328,14 @@ public abstract class AbstractCSQueue implements CSQueue {
return this.queueNodeLabelsSettings.getDefaultLabelExpression();
}
- protected void setupQueueConfigs(Resource clusterResource,
- CapacitySchedulerConfiguration configuration) throws
+ protected void setupQueueConfigs(Resource clusterResource) throws
IOException {
writeLock.lock();
try {
+ CapacitySchedulerConfiguration configuration =
queueContext.getConfiguration();
if (isDynamicQueue() || this instanceof AbstractAutoCreatedLeafQueue) {
-setDynamicQueueProperties(configuration);
+setDynamicQueueProperties();
}
// Collect and set the Node label configuration
@@ -344,7 +343,7 @@ public abstract class AbstractCSQueue implements CSQueue {
getQueuePath(),
queueContext.getQueueManager().getConfiguredNodeLabelsForAllQueues());
// Initialize the queue capacities
- setupConfigurableCapacities(configuration);
+ setupConfigurableCapacities();
updateAbsoluteCapacities();
updateCapacityConfigType();
@@ -354,26 +353,23 @@ public abstract class AbstractCSQueue implements CSQueue {
// Setup queue's maximumAllocation respecting the global
// and the queue settings
- // TODO remove the getConfiguration() param after the AQC configuration
duplication
- // removal is resolved
- this.queueAllocationSettings.setupMaximumAllocation(configuration,
- queueContext.getConfiguration(), getQueuePath(),
+ this.queueAllocationSettings.setupMaximumAllocation(configuration,
getQueuePath(),
parent);
// Initialize the queue state based on previous state, configured state
// and its parent state
- initializeQueueState(configuration);
+ initializeQueueState();
authorizer = YarnAuthorizationProvider.getInstance(configuration);
this.acls = configuration.getAcls(getQueuePath());
- this.userWeights = getUserWeightsFromHierarchy(configuration);
+ this.userWeights = getUserWeightsFromHierarchy();
[hadoop] branch trunk updated: HDFS-16014: Fix an issue in checking native pmdk lib by 'hadoop checknative' command (#3762)
This is an automated email from the ASF dual-hosted git repository.
rakeshr pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 8e08f43 HDFS-16014: Fix an issue in checking native pmdk lib by
'hadoop checknative' command (#3762)
8e08f43 is described below
commit 8e08f43e0386e355c6e32282e5f5eadc5904ec2d
Author: PHILO-HE
AuthorDate: Tue Dec 14 17:15:12 2021 +0800
HDFS-16014: Fix an issue in checking native pmdk lib by 'hadoop
checknative' command (#3762)
---
.../main/java/org/apache/hadoop/io/nativeio/NativeIO.java | 2 +-
.../native/src/org/apache/hadoop/io/nativeio/pmdk_load.c | 15 ---
2 files changed, 13 insertions(+), 4 deletions(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
index 79b489b..ebe7f21 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
@@ -141,7 +141,7 @@ public class NativeIO {
}
}
-// Denotes the state of supporting PMDK. The value is set by JNI.
+// Denotes the state of supporting PMDK. The actual value is set via JNI.
private static SupportState pmdkSupportState =
SupportState.UNSUPPORTED;
diff --git
a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/pmdk_load.c
b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/pmdk_load.c
index 502508c..f1a1df5 100644
---
a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/pmdk_load.c
+++
b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/pmdk_load.c
@@ -35,13 +35,14 @@
#endif
PmdkLibLoader * pmdkLoader;
+// 1 represents loaded. Otherwise, not loaded.
+int pmdkLoaded;
/**
* pmdk_load.c
* Utility of loading the libpmem library and the required functions.
* Building of this codes won't rely on any libpmem source codes, but running
* into this will rely on successfully loading of the dynamic library.
- *
*/
static const char* load_functions() {
@@ -56,6 +57,10 @@ static const char* load_functions() {
return NULL;
}
+/**
+ * It should be idempotent to call this function for checking
+ * whether PMDK lib is successfully loaded.
+ */
void load_pmdk_lib(char* err, size_t err_len) {
const char* errMsg;
const char* library = NULL;
@@ -67,10 +72,13 @@ void load_pmdk_lib(char* err, size_t err_len) {
err[0] = '\0';
- if (pmdkLoader != NULL) {
+ if (pmdkLoaded == 1) {
return;
}
- pmdkLoader = calloc(1, sizeof(PmdkLibLoader));
+
+ if (pmdkLoader == NULL) {
+pmdkLoader = calloc(1, sizeof(PmdkLibLoader));
+ }
// Load PMDK library
#ifdef UNIX
@@ -103,4 +111,5 @@ void load_pmdk_lib(char* err, size_t err_len) {
}
pmdkLoader->libname = strdup(library);
+ pmdkLoaded = 1;
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.2 updated: HDFS-16373. Fix MiniDFSCluster restart in case of multiple namenodes. (#3756)
This is an automated email from the ASF dual-hosted git repository.
tasanuma pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new a57e83a HDFS-16373. Fix MiniDFSCluster restart in case of multiple
namenodes. (#3756)
a57e83a is described below
commit a57e83a47ba9322ee4a21dd1a66001f7f30851ba
Author: Ayush Saxena
AuthorDate: Tue Dec 14 13:51:51 2021 +0530
HDFS-16373. Fix MiniDFSCluster restart in case of multiple namenodes.
(#3756)
Reviewed-by: Viraj Jasani
Reviewed-by: litao
Signed-off-by: Takanobu Asanuma
---
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 43 +++---
.../org/apache/hadoop/hdfs/TestMiniDFSCluster.java | 8
2 files changed, 30 insertions(+), 21 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 1aff916..c5c33b0 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -2234,9 +2234,11 @@ public class MiniDFSCluster implements AutoCloseable {
info.nameNode = nn;
info.setStartOpt(startOpt);
if (waitActive) {
- waitClusterUp();
+ if (numDataNodes > 0) {
+waitNameNodeUp(nnIndex);
+ }
LOG.info("Restarted the namenode");
- waitActive();
+ waitActive(nnIndex);
}
}
@@ -2746,11 +2748,25 @@ public class MiniDFSCluster implements AutoCloseable {
DFSClient client = new DFSClient(addr, conf);
// ensure all datanodes have registered and sent heartbeat to the namenode
-while (shouldWait(client.datanodeReport(DatanodeReportType.LIVE), addr)) {
+int failedCount = 0;
+while (true) {
try {
-LOG.info("Waiting for cluster to become active");
-Thread.sleep(100);
+while (shouldWait(client.datanodeReport(DatanodeReportType.LIVE),
addr)) {
+ LOG.info("Waiting for cluster to become active");
+ Thread.sleep(100);
+}
+break;
+ } catch (IOException e) {
+failedCount++;
+// Cached RPC connection to namenode, if any, is expected to fail once
+if (failedCount > 1) {
+ LOG.warn("Tried waitActive() " + failedCount
+ + " time(s) and failed, giving up. " + StringUtils
+ .stringifyException(e));
+ throw e;
+}
} catch (InterruptedException e) {
+throw new IOException(e);
}
}
@@ -2786,22 +2802,7 @@ public class MiniDFSCluster implements AutoCloseable {
*/
public void waitActive() throws IOException {
for (int index = 0; index < namenodes.size(); index++) {
- int failedCount = 0;
- while (true) {
-try {
- waitActive(index);
- break;
-} catch (IOException e) {
- failedCount++;
- // Cached RPC connection to namenode, if any, is expected to fail
once
- if (failedCount > 1) {
-LOG.warn("Tried waitActive() " + failedCount
-+ " time(s) and failed, giving up. "
-+ StringUtils.stringifyException(e));
-throw e;
- }
-}
- }
+ waitActive(index);
}
LOG.info("Cluster is active");
}
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
index d60e025..6403cc0 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
@@ -309,6 +309,14 @@ public class TestMiniDFSCluster {
DFSUtil.addKeySuffixes(
DFS_NAMENODE_HTTP_ADDRESS_KEY, "ns1", "nn1")));
}
+
+ // Shutdown namenodes individually.
+ cluster.shutdownNameNode(0);
+ cluster.shutdownNameNode(1);
+
+ // Restart namenodes individually with wait active, both should be
successful.
+ cluster.restartNameNode(0);
+ cluster.restartNameNode(1);
}
}
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.3 updated: HDFS-16373. Fix MiniDFSCluster restart in case of multiple namenodes. (#3756)
This is an automated email from the ASF dual-hosted git repository.
tasanuma pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 37ced45 HDFS-16373. Fix MiniDFSCluster restart in case of multiple
namenodes. (#3756)
37ced45 is described below
commit 37ced452d99e33b942074bfeac65e22ecd037a16
Author: Ayush Saxena
AuthorDate: Tue Dec 14 13:51:51 2021 +0530
HDFS-16373. Fix MiniDFSCluster restart in case of multiple namenodes.
(#3756)
Reviewed-by: Viraj Jasani
Reviewed-by: litao
Signed-off-by: Takanobu Asanuma
---
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 43 +++---
.../org/apache/hadoop/hdfs/TestMiniDFSCluster.java | 8
2 files changed, 30 insertions(+), 21 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index ff38236..6d2dabf 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -2249,9 +2249,11 @@ public class MiniDFSCluster implements AutoCloseable {
info.nameNode = nn;
info.setStartOpt(startOpt);
if (waitActive) {
- waitClusterUp();
+ if (numDataNodes > 0) {
+waitNameNodeUp(nnIndex);
+ }
LOG.info("Restarted the namenode");
- waitActive();
+ waitActive(nnIndex);
}
}
@@ -2761,11 +2763,25 @@ public class MiniDFSCluster implements AutoCloseable {
DFSClient client = new DFSClient(addr, conf);
// ensure all datanodes have registered and sent heartbeat to the namenode
-while (shouldWait(client.datanodeReport(DatanodeReportType.LIVE), addr)) {
+int failedCount = 0;
+while (true) {
try {
-LOG.info("Waiting for cluster to become active");
-Thread.sleep(100);
+while (shouldWait(client.datanodeReport(DatanodeReportType.LIVE),
addr)) {
+ LOG.info("Waiting for cluster to become active");
+ Thread.sleep(100);
+}
+break;
+ } catch (IOException e) {
+failedCount++;
+// Cached RPC connection to namenode, if any, is expected to fail once
+if (failedCount > 1) {
+ LOG.warn("Tried waitActive() " + failedCount
+ + " time(s) and failed, giving up. " + StringUtils
+ .stringifyException(e));
+ throw e;
+}
} catch (InterruptedException e) {
+throw new IOException(e);
}
}
@@ -2801,22 +2817,7 @@ public class MiniDFSCluster implements AutoCloseable {
*/
public void waitActive() throws IOException {
for (int index = 0; index < namenodes.size(); index++) {
- int failedCount = 0;
- while (true) {
-try {
- waitActive(index);
- break;
-} catch (IOException e) {
- failedCount++;
- // Cached RPC connection to namenode, if any, is expected to fail
once
- if (failedCount > 1) {
-LOG.warn("Tried waitActive() " + failedCount
-+ " time(s) and failed, giving up. "
-+ StringUtils.stringifyException(e));
-throw e;
- }
-}
- }
+ waitActive(index);
}
LOG.info("Cluster is active");
}
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
index 74a8e44..6b428c5 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
@@ -309,6 +309,14 @@ public class TestMiniDFSCluster {
DFSUtil.addKeySuffixes(
DFS_NAMENODE_HTTP_ADDRESS_KEY, "ns1", "nn1")));
}
+
+ // Shutdown namenodes individually.
+ cluster.shutdownNameNode(0);
+ cluster.shutdownNameNode(1);
+
+ // Restart namenodes individually with wait active, both should be
successful.
+ cluster.restartNameNode(0);
+ cluster.restartNameNode(1);
}
}
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: HDFS-16373. Fix MiniDFSCluster restart in case of multiple namenodes. (#3756)
This is an automated email from the ASF dual-hosted git repository.
tasanuma pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new d29f0e8 HDFS-16373. Fix MiniDFSCluster restart in case of multiple
namenodes. (#3756)
d29f0e8 is described below
commit d29f0e83a935a0443537064d1e7290ea67301045
Author: Ayush Saxena
AuthorDate: Tue Dec 14 13:51:51 2021 +0530
HDFS-16373. Fix MiniDFSCluster restart in case of multiple namenodes.
(#3756)
Reviewed-by: Viraj Jasani
Reviewed-by: litao
Signed-off-by: Takanobu Asanuma
---
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 43 +++---
.../org/apache/hadoop/hdfs/TestMiniDFSCluster.java | 8
2 files changed, 30 insertions(+), 21 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index e4b6434..703111e 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -2267,9 +2267,11 @@ public class MiniDFSCluster implements AutoCloseable {
info.nameNode = nn;
info.setStartOpt(startOpt);
if (waitActive) {
- waitClusterUp();
+ if (numDataNodes > 0) {
+waitNameNodeUp(nnIndex);
+ }
LOG.info("Restarted the namenode");
- waitActive();
+ waitActive(nnIndex);
}
}
@@ -2775,11 +2777,25 @@ public class MiniDFSCluster implements AutoCloseable {
DFSClient client = new DFSClient(addr, conf);
// ensure all datanodes have registered and sent heartbeat to the namenode
-while (shouldWait(client.datanodeReport(DatanodeReportType.LIVE), addr)) {
+int failedCount = 0;
+while (true) {
try {
-LOG.info("Waiting for cluster to become active");
-Thread.sleep(100);
+while (shouldWait(client.datanodeReport(DatanodeReportType.LIVE),
addr)) {
+ LOG.info("Waiting for cluster to become active");
+ Thread.sleep(100);
+}
+break;
+ } catch (IOException e) {
+failedCount++;
+// Cached RPC connection to namenode, if any, is expected to fail once
+if (failedCount > 1) {
+ LOG.warn("Tried waitActive() " + failedCount
+ + " time(s) and failed, giving up. " + StringUtils
+ .stringifyException(e));
+ throw e;
+}
} catch (InterruptedException e) {
+throw new IOException(e);
}
}
@@ -2815,22 +2831,7 @@ public class MiniDFSCluster implements AutoCloseable {
*/
public void waitActive() throws IOException {
for (int index = 0; index < namenodes.size(); index++) {
- int failedCount = 0;
- while (true) {
-try {
- waitActive(index);
- break;
-} catch (IOException e) {
- failedCount++;
- // Cached RPC connection to namenode, if any, is expected to fail
once
- if (failedCount > 1) {
-LOG.warn("Tried waitActive() " + failedCount
-+ " time(s) and failed, giving up. "
-+ StringUtils.stringifyException(e));
-throw e;
- }
-}
- }
+ waitActive(index);
}
LOG.info("Cluster is active");
}
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
index 74cfe9d..737795b 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
@@ -309,6 +309,14 @@ public class TestMiniDFSCluster {
DFSUtil.addKeySuffixes(
DFS_NAMENODE_HTTP_ADDRESS_KEY, "ns1", "nn1")));
}
+
+ // Shutdown namenodes individually.
+ cluster.shutdownNameNode(0);
+ cluster.shutdownNameNode(1);
+
+ // Restart namenodes individually with wait active, both should be
successful.
+ cluster.restartNameNode(0);
+ cluster.restartNameNode(1);
}
}
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
