(hadoop) branch HDFS-17384 updated: HDFS-17417. [FGL] HeartbeatManager and DatanodeAdminMonitor support fine-grained locking (#6656)
This is an automated email from the ASF dual-hosted git repository.
ferhui pushed a commit to branch HDFS-17384
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/HDFS-17384 by this push:
new 313f7058507a HDFS-17417. [FGL] HeartbeatManager and
DatanodeAdminMonitor support fine-grained locking (#6656)
313f7058507a is described below
commit 313f7058507a412b629bf643b48f8c3d13119f79
Author: ZanderXu
AuthorDate: Wed Mar 27 09:45:17 2024 +0800
HDFS-17417. [FGL] HeartbeatManager and DatanodeAdminMonitor support
fine-grained locking (#6656)
---
.../hdfs/server/blockmanagement/BlockManager.java | 10 +++---
.../DatanodeAdminBackoffMonitor.java | 38 --
.../DatanodeAdminDefaultMonitor.java | 11 ---
.../server/blockmanagement/DatanodeManager.java| 5 +--
.../server/blockmanagement/HeartbeatManager.java | 9 ++---
5 files changed, 41 insertions(+), 32 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 302392f19d4d..a0dc2ee70fbf 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -1825,7 +1825,7 @@ public class BlockManager implements BlockStatsMXBean {
/** Remove the blocks associated to the given DatanodeStorageInfo. */
void removeBlocksAssociatedTo(final DatanodeStorageInfo storageInfo) {
-assert namesystem.hasWriteLock();
+assert namesystem.hasWriteLock(FSNamesystemLockMode.BM);
final Iterator it = storageInfo.getBlockIterator();
DatanodeDescriptor node = storageInfo.getDatanodeDescriptor();
while(it.hasNext()) {
@@ -4864,6 +4864,7 @@ public class BlockManager implements BlockStatsMXBean {
NumberReplicas num = countNodes(block);
if (shouldProcessExtraRedundancy(num, expectedReplication)) {
// extra redundancy block
+ // Here involves storage policy ID.
processExtraRedundancyBlock(block, (short) expectedReplication, null,
null);
numExtraRedundancy++;
@@ -4872,14 +4873,15 @@ public class BlockManager implements BlockStatsMXBean {
// When called by tests like TestDefaultBlockPlacementPolicy.
// testPlacementWithLocalRackNodesDecommissioned, it is not protected by
// lock, only when called by DatanodeManager.refreshNodes have writeLock
- if (namesystem.hasWriteLock()) {
-namesystem.writeUnlock("processExtraRedundancyBlocksOnInService");
+ if (namesystem.hasWriteLock(FSNamesystemLockMode.GLOBAL)) {
+namesystem.writeUnlock(FSNamesystemLockMode.GLOBAL,
+"processExtraRedundancyBlocksOnInService");
try {
Thread.sleep(1);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
-namesystem.writeLock();
+namesystem.writeLock(FSNamesystemLockMode.GLOBAL);
}
}
LOG.info("Invalidated {} extra redundancy blocks on {} after "
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminBackoffMonitor.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminBackoffMonitor.java
index 79d5a065b08a..d212d142d441 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminBackoffMonitor.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminBackoffMonitor.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
+import org.apache.hadoop.hdfs.server.namenode.fgl.FSNamesystemLockMode;
import org.apache.hadoop.thirdparty.com.google.common.collect.Iterables;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.server.namenode.INode;
@@ -170,7 +171,7 @@ public class DatanodeAdminBackoffMonitor extends
DatanodeAdminMonitorBase
numBlocksChecked = 0;
// Check decommission or maintenance progress.
try {
- namesystem.writeLock();
+ namesystem.writeLock(FSNamesystemLockMode.BM);
try {
/**
* Other threads can modify the pendingNode list and the cancelled
@@ -208,7 +209,7 @@ public class DatanodeAdminBackoffMonitor extends
DatanodeAdminMonitorBase
processPendingNodes();
} finally {
-namesystem.writeUnlock("DatanodeAdminMonitorV2Thread");
+namesystem.writeUnlock(FSNamesystemLockMode.BM,
"DatanodeAdminMonitorV2Thread");
}
// After processing the above, various parts of the check()
(hadoop) branch HDFS-17384 updated: HDFS-17415. [FGL] RPCs in NamenodeProtocol support fine-grained lock (#6654)
This is an automated email from the ASF dual-hosted git repository.
ferhui pushed a commit to branch HDFS-17384
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/HDFS-17384 by this push:
new 3ad137d90e25 HDFS-17415. [FGL] RPCs in NamenodeProtocol support
fine-grained lock (#6654)
3ad137d90e25 is described below
commit 3ad137d90e25c5f1ef56e551aee0924010eb3b72
Author: ZanderXu
AuthorDate: Wed Mar 27 09:44:11 2024 +0800
HDFS-17415. [FGL] RPCs in NamenodeProtocol support fine-grained lock (#6654)
---
.../hadoop/hdfs/server/namenode/Checkpointer.java | 10 +
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 25 +++---
2 files changed, 19 insertions(+), 16 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
index ab657fb67249..465aad3ec50b 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile;
+import org.apache.hadoop.hdfs.server.namenode.fgl.FSNamesystemLockMode;
import org.apache.hadoop.hdfs.server.protocol.CheckpointCommand;
import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
@@ -244,13 +245,14 @@ class Checkpointer extends Daemon {
if(needReloadImage) {
LOG.info("Loading image with txid " + sig.mostRecentCheckpointTxId);
-backupNode.namesystem.writeLock();
+backupNode.namesystem.writeLock(FSNamesystemLockMode.GLOBAL);
try {
File file = bnStorage.findImageFile(NameNodeFile.IMAGE,
sig.mostRecentCheckpointTxId);
bnImage.reloadFromImageFile(file, backupNode.getNamesystem());
} finally {
- backupNode.namesystem.writeUnlock("doCheckpointByBackupNode");
+ backupNode.namesystem.writeUnlock(
+ FSNamesystemLockMode.GLOBAL, "doCheckpointByBackupNode");
}
}
rollForwardByApplyingLogs(manifest, bnImage, backupNode.getNamesystem());
@@ -258,7 +260,7 @@ class Checkpointer extends Daemon {
long txid = bnImage.getLastAppliedTxId();
-backupNode.namesystem.writeLock();
+backupNode.namesystem.writeLock(FSNamesystemLockMode.GLOBAL);
try {
backupNode.namesystem.setImageLoaded();
if(backupNode.namesystem.getBlocksTotal() > 0) {
@@ -272,7 +274,7 @@ class Checkpointer extends Daemon {
bnImage.updateStorageVersion();
}
} finally {
- backupNode.namesystem.writeUnlock("doCheckpoint");
+ backupNode.namesystem.writeUnlock(FSNamesystemLockMode.GLOBAL,
"doCheckpoint");
}
if(cpCmd.needToReturnImage()) {
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 3610ca4b9621..df7e2bcc2dab 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -1973,13 +1973,13 @@ public class FSNamesystem implements Namesystem,
FSNamesystemMBean,
isGetBlocksCheckOperationEnabled ? OperationCategory.READ :
OperationCategory.UNCHECKED;
checkOperation(checkOp);
-readLock();
+readLock(FSNamesystemLockMode.BM);
try {
checkOperation(checkOp);
return getBlockManager().getBlocksWithLocations(datanode, size,
minimumBlockSize, timeInterval, storageType);
} finally {
- readUnlock("getBlocks");
+ readUnlock(FSNamesystemLockMode.BM, "getBlocks");
}
}
@@ -5314,7 +5314,8 @@ public class FSNamesystem implements Namesystem,
FSNamesystemMBean,
CheckpointSignature result = null;
checkSuperuserPrivilege(operationName);
checkOperation(OperationCategory.JOURNAL);
-writeLock();
+// Using FSWriteLock to make edit-related operations thread safe.
+writeLock(FSNamesystemLockMode.FS);
try {
checkOperation(OperationCategory.JOURNAL);
checkNameNodeSafeMode("Log not rolled");
@@ -5323,7 +5324,7 @@ public class FSNamesystem implements Namesystem,
FSNamesystemMBean,
}
result = getFSImage().rollEditLog(getEffectiveLayoutVersion());
} finally {
- writeUnlock(operationName,
(hadoop) branch HDFS-17384 updated: HDFS-17410. [FGL] Client RPCs that changes file attributes supports fine-grained lock (#6634)
This is an automated email from the ASF dual-hosted git repository.
ferhui pushed a commit to branch HDFS-17384
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/HDFS-17384 by this push:
new 123915508b61 HDFS-17410. [FGL] Client RPCs that changes file
attributes supports fine-grained lock (#6634)
123915508b61 is described below
commit 123915508b61978ac448d6951184194ea58e35d4
Author: ZanderXu
AuthorDate: Wed Mar 27 09:41:17 2024 +0800
HDFS-17410. [FGL] Client RPCs that changes file attributes supports
fine-grained lock (#6634)
---
.../namenode/ContentSummaryComputationContext.java | 9 +-
.../server/namenode/EncryptionZoneManager.java | 14 +--
.../server/namenode/FSDirEncryptionZoneOp.java | 7 +-
.../namenode/FSDirSatisfyStoragePolicyOp.java | 3 +-
.../hdfs/server/namenode/FSDirTruncateOp.java | 7 +-
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 116 +++--
.../hdfs/server/namenode/ReencryptionHandler.java | 11 +-
.../hdfs/server/namenode/ReencryptionUpdater.java | 7 +-
8 files changed, 92 insertions(+), 82 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
index bbebe78fb1f6..c7b510eca565 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import org.apache.hadoop.hdfs.server.namenode.fgl.FSNamesystemLockMode;
import org.apache.hadoop.util.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -118,8 +119,8 @@ public class ContentSummaryComputationContext {
boolean hadDirReadLock = dir.hasReadLock();
boolean hadDirWriteLock = dir.hasWriteLock();
-boolean hadFsnReadLock = fsn.hasReadLock();
-boolean hadFsnWriteLock = fsn.hasWriteLock();
+boolean hadFsnReadLock = fsn.hasReadLock(FSNamesystemLockMode.GLOBAL);
+boolean hadFsnWriteLock = fsn.hasWriteLock(FSNamesystemLockMode.GLOBAL);
// sanity check.
if (!hadDirReadLock || !hadFsnReadLock || hadDirWriteLock ||
@@ -130,14 +131,14 @@ public class ContentSummaryComputationContext {
// unlock
dir.readUnlock();
-fsn.readUnlock("contentSummary");
+fsn.readUnlock(FSNamesystemLockMode.GLOBAL, "contentSummary");
try {
Thread.sleep(sleepMilliSec, sleepNanoSec);
} catch (InterruptedException ie) {
} finally {
// reacquire
- fsn.readLock();
+ fsn.readLock(FSNamesystemLockMode.GLOBAL);
dir.readLock();
}
yieldCount++;
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
index 7bf587997161..1f75a7734cb4 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
+import org.apache.hadoop.hdfs.server.namenode.fgl.FSNamesystemLockMode;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.util.Lists;
@@ -188,11 +189,12 @@ public class EncryptionZoneManager {
final int count) throws IOException {
INodesInPath iip;
final FSPermissionChecker pc = dir.getPermissionChecker();
-dir.getFSNamesystem().readLock();
+dir.getFSNamesystem().readLock(FSNamesystemLockMode.FS);
try {
iip = dir.resolvePath(pc, zone, DirOp.READ);
} finally {
- dir.getFSNamesystem().readUnlock("pauseForTestingAfterNthCheckpoint");
+ dir.getFSNamesystem().readUnlock(
+ FSNamesystemLockMode.FS, "pauseForTestingAfterNthCheckpoint");
}
reencryptionHandler
.pauseForTestingAfterNthCheckpoint(iip.getLastINode().getId(), count);
@@ -213,7 +215,7 @@ public class EncryptionZoneManager {
throws IOException {
final FSPermissionChecker pc = dir.getPermissionChecker();
final INode inode;
-dir.getFSNamesystem().readLock();
+
(hadoop) branch trunk updated: HDFS-17429. Fixing wrong log file name in datatransfer Sender.java (#6670) Contributed by Zhongkun Wu.
This is an automated email from the ASF dual-hosted git repository.
slfan1989 pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 50370769cd56 HDFS-17429. Fixing wrong log file name in datatransfer
Sender.java (#6670) Contributed by Zhongkun Wu.
50370769cd56 is described below
commit 50370769cd56f37d3bf7698d65e9044c264e7d06
Author: Alex
AuthorDate: Wed Mar 27 06:59:15 2024 +0800
HDFS-17429. Fixing wrong log file name in datatransfer Sender.java (#6670)
Contributed by Zhongkun Wu.
Reviewed-by: Ayush Saxena
Signed-off-by: Shilun Fan
---
.../java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java| 5 +
1 file changed, 5 insertions(+)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java
index 3d81a62993ef..86c6513745ea 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java
@@ -57,11 +57,16 @@ import org.apache.hadoop.tracing.Tracer;
import org.apache.hadoop.tracing.TraceUtils;
import org.apache.hadoop.thirdparty.protobuf.Message;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/** Sender */
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class Sender implements DataTransferProtocol {
+ private static final Logger LOG = LoggerFactory.getLogger(Sender.class);
+
private final DataOutputStream out;
/** Create a sender for DataTransferProtocol with a output stream. */
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
(hadoop) branch branch-3.4 updated: HADOOP-19047: S3A: Support in-memory tracking of Magic Commit data (#6468)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new f2ea733732f1 HADOOP-19047: S3A: Support in-memory tracking of Magic
Commit data (#6468)
f2ea733732f1 is described below
commit f2ea733732f1aa50e81df8ab013aa21ab36053b8
Author: Syed Shameerur Rahman
AuthorDate: Tue Mar 26 20:59:35 2024 +0530
HADOOP-19047: S3A: Support in-memory tracking of Magic Commit data (#6468)
If the option fs.s3a.committer.magic.track.commits.in.memory.enabled
is set to true, then rather than save data about in-progress uploads
to S3, this information is cached in memory.
If the number of files being committed is low, this will save network IO
in both the generation of .pending and marker files, and in the scanning
of task attempt directory trees during task commit.
Contributed by Syed Shameerur Rahman
---
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java| 18 +++
.../hadoop/fs/s3a/commit/CommitConstants.java | 16 +++
.../fs/s3a/commit/MagicCommitIntegration.java | 20 +--
.../commit/magic/InMemoryMagicCommitTracker.java | 146 +
.../fs/s3a/commit/magic/MagicCommitTracker.java| 106 ---
.../s3a/commit/magic/MagicCommitTrackerUtils.java | 64 +
.../fs/s3a/commit/magic/MagicS3GuardCommitter.java | 108 ---
...ommitTracker.java => S3MagicCommitTracker.java} | 109 +++
.../site/markdown/tools/hadoop-aws/committers.md | 7 +
.../fs/s3a/commit/AbstractITCommitProtocol.java| 15 ++-
.../fs/s3a/commit/TestMagicCommitTrackerUtils.java | 64 +
.../s3a/commit/magic/ITestMagicCommitProtocol.java | 30 +
.../fs/s3a/commit/terasort/ITestTerasortOnS3A.java | 17 ++-
13 files changed, 516 insertions(+), 204 deletions(-)
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 3aec03766dac..755f1fffbdb1 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -117,6 +117,7 @@ import org.apache.hadoop.fs.s3a.audit.AuditSpanS3A;
import org.apache.hadoop.fs.s3a.auth.SignerManager;
import org.apache.hadoop.fs.s3a.auth.delegation.DelegationOperations;
import org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenProvider;
+import org.apache.hadoop.fs.s3a.commit.magic.InMemoryMagicCommitTracker;
import org.apache.hadoop.fs.s3a.impl.AWSCannedACL;
import org.apache.hadoop.fs.s3a.impl.AWSHeaders;
import org.apache.hadoop.fs.s3a.impl.BulkDeleteRetryHandler;
@@ -231,6 +232,8 @@ import static
org.apache.hadoop.fs.s3a.auth.delegation.S3ADelegationTokens.Token
import static
org.apache.hadoop.fs.s3a.auth.delegation.S3ADelegationTokens.hasDelegationTokenBinding;
import static
org.apache.hadoop.fs.s3a.commit.CommitConstants.FS_S3A_COMMITTER_ABORT_PENDING_UPLOADS;
import static
org.apache.hadoop.fs.s3a.commit.CommitConstants.FS_S3A_COMMITTER_STAGING_ABORT_PENDING_UPLOADS;
+import static
org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC_COMMITTER_PENDING_OBJECT_ETAG_NAME;
+import static
org.apache.hadoop.fs.s3a.commit.magic.MagicCommitTrackerUtils.isTrackMagicCommitsInMemoryEnabled;
import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.submit;
import static
org.apache.hadoop.fs.s3a.impl.CreateFileBuilder.OPTIONS_CREATE_FILE_NO_OVERWRITE;
import static
org.apache.hadoop.fs.s3a.impl.CreateFileBuilder.OPTIONS_CREATE_FILE_OVERWRITE;
@@ -3894,6 +3897,21 @@ public class S3AFileSystem extends FileSystem implements
StreamCapabilities,
@Retries.RetryTranslated
public FileStatus getFileStatus(final Path f) throws IOException {
Path path = qualify(f);
+if (isTrackMagicCommitsInMemoryEnabled(getConf()) &&
isMagicCommitPath(path)) {
+ // Some downstream apps might call getFileStatus for a magic path to get
the file size.
+ // when commit data is stored in memory construct the dummy
S3AFileStatus with correct
+ // file size fetched from the memory.
+ if
(InMemoryMagicCommitTracker.getPathToBytesWritten().containsKey(path)) {
+long len =
InMemoryMagicCommitTracker.getPathToBytesWritten().get(path);
+return new S3AFileStatus(len,
+0L,
+path,
+getDefaultBlockSize(path),
+username,
+MAGIC_COMMITTER_PENDING_OBJECT_ETAG_NAME,
+null);
+ }
+}
return trackDurationAndSpan(
INVOCATION_GET_FILE_STATUS, path, () ->
innerGetFileStatus(path, false, StatusProbeEnum.ALL));
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java
(hadoop) branch trunk updated: HADOOP-19047: S3A: Support in-memory tracking of Magic Commit data (#6468)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 032796a0fb21 HADOOP-19047: S3A: Support in-memory tracking of Magic
Commit data (#6468)
032796a0fb21 is described below
commit 032796a0fb21b432b8d873902fe00b44afa2496c
Author: Syed Shameerur Rahman
AuthorDate: Tue Mar 26 20:59:35 2024 +0530
HADOOP-19047: S3A: Support in-memory tracking of Magic Commit data (#6468)
If the option fs.s3a.committer.magic.track.commits.in.memory.enabled
is set to true, then rather than save data about in-progress uploads
to S3, this information is cached in memory.
If the number of files being committed is low, this will save network IO
in both the generation of .pending and marker files, and in the scanning
of task attempt directory trees during task commit.
Contributed by Syed Shameerur Rahman
---
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java| 18 +++
.../hadoop/fs/s3a/commit/CommitConstants.java | 16 +++
.../fs/s3a/commit/MagicCommitIntegration.java | 20 +--
.../commit/magic/InMemoryMagicCommitTracker.java | 146 +
.../fs/s3a/commit/magic/MagicCommitTracker.java| 106 ---
.../s3a/commit/magic/MagicCommitTrackerUtils.java | 64 +
.../fs/s3a/commit/magic/MagicS3GuardCommitter.java | 108 ---
...ommitTracker.java => S3MagicCommitTracker.java} | 109 +++
.../site/markdown/tools/hadoop-aws/committers.md | 7 +
.../fs/s3a/commit/AbstractITCommitProtocol.java| 15 ++-
.../fs/s3a/commit/TestMagicCommitTrackerUtils.java | 64 +
.../s3a/commit/magic/ITestMagicCommitProtocol.java | 30 +
.../fs/s3a/commit/terasort/ITestTerasortOnS3A.java | 17 ++-
13 files changed, 516 insertions(+), 204 deletions(-)
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index a404fc1c2173..0e2ae0f74dd0 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -117,6 +117,7 @@ import org.apache.hadoop.fs.s3a.audit.AuditSpanS3A;
import org.apache.hadoop.fs.s3a.auth.SignerManager;
import org.apache.hadoop.fs.s3a.auth.delegation.DelegationOperations;
import org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenProvider;
+import org.apache.hadoop.fs.s3a.commit.magic.InMemoryMagicCommitTracker;
import org.apache.hadoop.fs.s3a.impl.AWSCannedACL;
import org.apache.hadoop.fs.s3a.impl.AWSHeaders;
import org.apache.hadoop.fs.s3a.impl.BulkDeleteRetryHandler;
@@ -231,6 +232,8 @@ import static
org.apache.hadoop.fs.s3a.auth.delegation.S3ADelegationTokens.Token
import static
org.apache.hadoop.fs.s3a.auth.delegation.S3ADelegationTokens.hasDelegationTokenBinding;
import static
org.apache.hadoop.fs.s3a.commit.CommitConstants.FS_S3A_COMMITTER_ABORT_PENDING_UPLOADS;
import static
org.apache.hadoop.fs.s3a.commit.CommitConstants.FS_S3A_COMMITTER_STAGING_ABORT_PENDING_UPLOADS;
+import static
org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC_COMMITTER_PENDING_OBJECT_ETAG_NAME;
+import static
org.apache.hadoop.fs.s3a.commit.magic.MagicCommitTrackerUtils.isTrackMagicCommitsInMemoryEnabled;
import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.submit;
import static
org.apache.hadoop.fs.s3a.impl.CreateFileBuilder.OPTIONS_CREATE_FILE_NO_OVERWRITE;
import static
org.apache.hadoop.fs.s3a.impl.CreateFileBuilder.OPTIONS_CREATE_FILE_OVERWRITE;
@@ -3900,6 +3903,21 @@ public class S3AFileSystem extends FileSystem implements
StreamCapabilities,
@Retries.RetryTranslated
public FileStatus getFileStatus(final Path f) throws IOException {
Path path = qualify(f);
+if (isTrackMagicCommitsInMemoryEnabled(getConf()) &&
isMagicCommitPath(path)) {
+ // Some downstream apps might call getFileStatus for a magic path to get
the file size.
+ // when commit data is stored in memory construct the dummy
S3AFileStatus with correct
+ // file size fetched from the memory.
+ if
(InMemoryMagicCommitTracker.getPathToBytesWritten().containsKey(path)) {
+long len =
InMemoryMagicCommitTracker.getPathToBytesWritten().get(path);
+return new S3AFileStatus(len,
+0L,
+path,
+getDefaultBlockSize(path),
+username,
+MAGIC_COMMITTER_PENDING_OBJECT_ETAG_NAME,
+null);
+ }
+}
return trackDurationAndSpan(
INVOCATION_GET_FILE_STATUS, path, () ->
innerGetFileStatus(path, false, StatusProbeEnum.ALL));
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java
(hadoop) branch branch-3.4 updated: HADOOP-18980. Invalid inputs for getTrimmedStringCollectionSplitByEquals (ADDENDUM) (#6546)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new b419c3a306d5 HADOOP-18980. Invalid inputs for
getTrimmedStringCollectionSplitByEquals (ADDENDUM) (#6546)
b419c3a306d5 is described below
commit b419c3a306d5b74c5a6d5873f5a7487838434a71
Author: Viraj Jasani
AuthorDate: Tue Mar 26 03:18:03 2024 -0800
HADOOP-18980. Invalid inputs for getTrimmedStringCollectionSplitByEquals
(ADDENDUM) (#6546)
This is a followup to #6406:
HADOOP-18980. S3A credential provider remapping: make extensible
It adds extra validation of key-value pairs in a configuration
option, with tests.
Contributed by Viraj Jasani
---
.../java/org/apache/hadoop/util/StringUtils.java | 28 +++-
.../org/apache/hadoop/util/TestStringUtils.java| 58 +++-
.../fs/s3a/TestS3AAWSCredentialsProvider.java | 77 --
3 files changed, 155 insertions(+), 8 deletions(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
index b8d999162d38..2585729950b5 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
@@ -40,6 +40,7 @@ import org.apache.commons.lang3.SystemUtils;
import org.apache.commons.lang3.time.FastDateFormat;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.net.NetUtils;
import org.apache.log4j.LogManager;
@@ -79,6 +80,18 @@ public class StringUtils {
public static final Pattern ENV_VAR_PATTERN = Shell.WINDOWS ?
WIN_ENV_VAR_PATTERN : SHELL_ENV_VAR_PATTERN;
+ /**
+ * {@link #getTrimmedStringCollectionSplitByEquals(String)} throws
+ * {@link IllegalArgumentException} with error message starting with this
string
+ * if the argument provided is not valid representation of non-empty
key-value
+ * pairs.
+ * Value = {@value}
+ */
+ @VisibleForTesting
+ public static final String STRING_COLLECTION_SPLIT_EQUALS_INVALID_ARG =
+ "Trimmed string split by equals does not correctly represent "
+ + "non-empty key-value pairs.";
+
/**
* Make a string representation of the exception.
* @param e The exception to stringify
@@ -494,10 +507,19 @@ public class StringUtils {
String[] trimmedList = getTrimmedStrings(str);
Map pairs = new HashMap<>();
for (String s : trimmedList) {
- String[] splitByKeyVal = getTrimmedStringsSplitByEquals(s);
- if (splitByKeyVal.length == 2) {
-pairs.put(splitByKeyVal[0], splitByKeyVal[1]);
+ if (s.isEmpty()) {
+continue;
}
+ String[] splitByKeyVal = getTrimmedStringsSplitByEquals(s);
+ Preconditions.checkArgument(
+ splitByKeyVal.length == 2,
+ STRING_COLLECTION_SPLIT_EQUALS_INVALID_ARG + " Input: " + str);
+ boolean emptyKey =
org.apache.commons.lang3.StringUtils.isEmpty(splitByKeyVal[0]);
+ boolean emptyVal =
org.apache.commons.lang3.StringUtils.isEmpty(splitByKeyVal[1]);
+ Preconditions.checkArgument(
+ !emptyKey && !emptyVal,
+ STRING_COLLECTION_SPLIT_EQUALS_INVALID_ARG + " Input: " + str);
+ pairs.put(splitByKeyVal[0], splitByKeyVal[1]);
}
return pairs;
}
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java
index d9bcf5842689..c9b42b07f4c9 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java
@@ -19,6 +19,9 @@
package org.apache.hadoop.util;
import java.util.Locale;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static
org.apache.hadoop.util.StringUtils.STRING_COLLECTION_SPLIT_EQUALS_INVALID_ARG;
import static
org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix.long2String;
import static
org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix.string2long;
import static org.junit.Assert.assertArrayEquals;
@@ -515,7 +518,7 @@ public class TestStringUtils extends UnitTestcaseTimeLimit {
}
@Test
- public void testStringCollectionSplitByEquals() {
+ public void testStringCollectionSplitByEqualsSuccess() {
Map splitMap =
StringUtils.getTrimmedStringCollectionSplitByEquals("");
Assertions
@@ -566,6 +569,59 @@
(hadoop) branch trunk updated (37f9ccdc868c -> 9fe371aa151a)
This is an automated email from the ASF dual-hosted git repository. stevel pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git from 37f9ccdc868c HDFS-17368. HA: Standby should exit safemode when resources are available. (#6518). Contributed by Zilong Zhu. add 9fe371aa151a HADOOP-18980. Invalid inputs for getTrimmedStringCollectionSplitByEquals (ADDENDUM) (#6546) No new revisions were added by this update. Summary of changes: .../java/org/apache/hadoop/util/StringUtils.java | 28 +++- .../org/apache/hadoop/util/TestStringUtils.java| 58 +++- .../fs/s3a/TestS3AAWSCredentialsProvider.java | 77 -- 3 files changed, 155 insertions(+), 8 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
(hadoop) branch trunk updated: HDFS-17368. HA: Standby should exit safemode when resources are available. (#6518). Contributed by Zilong Zhu.
This is an automated email from the ASF dual-hosted git repository.
hexiaoqiao pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 37f9ccdc868c HDFS-17368. HA: Standby should exit safemode when
resources are available. (#6518). Contributed by Zilong Zhu.
37f9ccdc868c is described below
commit 37f9ccdc868c28d03d59881b7fe3c5d3abbb5f6e
Author: Zilong Zhu
AuthorDate: Tue Mar 26 17:35:55 2024 +0800
HDFS-17368. HA: Standby should exit safemode when resources are available.
(#6518). Contributed by Zilong Zhu.
Signed-off-by: He Xiaoqiao
---
.../org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 11 +++
.../apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java | 7 +++
2 files changed, 14 insertions(+), 4 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 075398f000ec..542cc370465e 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -1578,6 +1578,10 @@ public class FSNamesystem implements Namesystem,
FSNamesystemMBean,
standbyCheckpointer = new StandbyCheckpointer(conf, this);
standbyCheckpointer.start();
}
+if (isNoManualAndResourceLowSafeMode()) {
+ LOG.info("Standby should not enter safe mode when resources are low,
exiting safe mode.");
+ leaveSafeMode(false);
+}
}
/**
@@ -5253,10 +5257,9 @@ public class FSNamesystem implements Namesystem,
FSNamesystemMBean,
String cmd = "Use \"hdfs dfsadmin -safemode leave\" to turn safe mode
off.";
synchronized (this) {
if (resourceLowSafeMode) {
-return "Resources are low on NN. Please add or free up more resources"
-+ "then turn off safe mode manually. NOTE: If you turn off safe "
-+ "mode before adding resources, the NN will immediately return to
"
-+ "safe mode. " + cmd;
+return "Resources are low on NN. Please add or free up more resources.
"
++ "NOTE: If you turn off safe mode before adding resources, the "
++ "NN will immediately return to safe mode. ";
} else if (manualSafeMode) {
return "It was turned on manually. " + cmd;
}
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
index 4766c4cecc9b..0ac9eae7f540 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
@@ -999,4 +999,11 @@ public class TestHASafeMode {
() -> miniCluster.transitionToObserver(0));
}
}
+
+ @Test
+ public void testTransitionToStandbyWhenSafeModeWithResourcesLow() throws
Exception {
+NameNodeAdapter.enterSafeMode(nn0, true);
+cluster.transitionToStandby(0);
+assertFalse("SNN should not enter safe mode when resources low",
nn0.isInSafeMode());
+ }
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
