[hadoop] branch trunk updated: HDFS-16440. RBF: Support router get HAServiceStatus with Lifeline RPC address (#3971)
This is an automated email from the ASF dual-hosted git repository.
inigoiri pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 48bef28 HDFS-16440. RBF: Support router get HAServiceStatus with
Lifeline RPC address (#3971)
48bef28 is described below
commit 48bef285a85e311b52335c8687c6e9a5f8afdf70
Author: YulongZ
AuthorDate: Wed Feb 16 00:44:17 2022 +0800
HDFS-16440. RBF: Support router get HAServiceStatus with Lifeline RPC
address (#3971)
---
.../router/NamenodeHeartbeatService.java | 10 +++-
.../router/TestRouterNamenodeHeartbeat.java| 56 --
2 files changed, 61 insertions(+), 5 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
index 8858341..e1c8a68 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
@@ -48,6 +48,8 @@ import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hadoop.classification.VisibleForTesting;
+
/**
* The {@link Router} periodically checks the state of a Namenode (usually on
* the same server) and reports their high availability (HA) state and
@@ -346,7 +348,8 @@ public class NamenodeHeartbeatService extends
PeriodicService {
// Determine if NN is active
// TODO: dynamic timeout
if (localTargetHAProtocol == null) {
-localTargetHAProtocol = localTarget.getProxy(conf, 30*1000);
+localTargetHAProtocol = localTarget.getHealthMonitorProxy(conf,
30*1000);
+LOG.debug("Get HA status with address {}", lifelineAddress);
}
HAServiceStatus status = localTargetHAProtocol.getServiceStatus();
report.setHAServiceState(status.getState());
@@ -373,6 +376,11 @@ public class NamenodeHeartbeatService extends
PeriodicService {
return report;
}
+ @VisibleForTesting
+ NNHAServiceTarget getLocalTarget(){
+return this.localTarget;
+ }
+
/**
* Get the description of the Namenode to monitor.
* @return Description of the Namenode to monitor.
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java
index 9bf149c..94f2bae 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java
@@ -212,6 +212,50 @@ public class TestRouterNamenodeHeartbeat {
}
@Test
+ public void testNamenodeHeartbeatServiceHAServiceProtocolProxy(){
+testNamenodeHeartbeatServiceHAServiceProtocol(
+"test-ns", "nn", 1000, -1, -1, 1003,
+"host01.test:1000", "host02.test:1000");
+testNamenodeHeartbeatServiceHAServiceProtocol(
+"test-ns", "nn", 1000, 1001, -1, 1003,
+"host01.test:1001", "host02.test:1001");
+testNamenodeHeartbeatServiceHAServiceProtocol(
+"test-ns", "nn", 1000, -1, 1002, 1003,
+"host01.test:1002", "host02.test:1002");
+testNamenodeHeartbeatServiceHAServiceProtocol(
+"test-ns", "nn", 1000, 1001, 1002, 1003,
+"host01.test:1002", "host02.test:1002");
+ }
+
+ private void testNamenodeHeartbeatServiceHAServiceProtocol(
+ String nsId, String nnId,
+ int rpcPort, int servicePort,
+ int lifelinePort, int webAddressPort,
+ String expected0, String expected1) {
+Configuration conf = generateNamenodeConfiguration(nsId, nnId,
+rpcPort, servicePort, lifelinePort, webAddressPort);
+
+Router testRouter = new Router();
+testRouter.setConf(conf);
+
+Collection heartbeatServices =
+testRouter.createNamenodeHeartbeatServices();
+
+assertEquals(2, heartbeatServices.size());
+
+Iterator iterator = heartbeatServices.iterator();
+NamenodeHeartbeatService service0 = iterator.next();
+service0.init(conf);
+assertNotNull(service0.getLocalTarget());
+assertEquals(expected0,
service0.getLocalTarget().getHealthMonitorAddress().toString());
+
+NamenodeHeartbeatService service1 = iterator.next();
+service1.init(conf);
+assertNotNull(service1.getLocalTarget());
+assertEquals(expected1,
service1.getLocalTarget().getHealth
[hadoop] branch branch-2.10 updated: HDFS-10650. DFSClient#mkdirs and DFSClient#primitiveMkdir should use default directory permission. Contributed by John Zhuge.
This is an automated email from the ASF dual-hosted git repository.
shv pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.10 by this push:
new cb5af00 HDFS-10650. DFSClient#mkdirs and DFSClient#primitiveMkdir
should use default directory permission. Contributed by John Zhuge.
cb5af00 is described below
commit cb5af0012ed94bb8ab63cfdcdfcd1ab17f6660bb
Author: Xiao Chen
AuthorDate: Thu Jul 28 13:15:02 2016 -0700
HDFS-10650. DFSClient#mkdirs and DFSClient#primitiveMkdir should use
default directory permission. Contributed by John Zhuge.
---
.../src/main/java/org/apache/hadoop/hdfs/DFSClient.java| 14 ++
.../java/org/apache/hadoop/security/TestPermission.java| 10 ++
2 files changed, 20 insertions(+), 4 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index ad4e499..32553fb 100755
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -1209,6 +1209,13 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
return permission.applyUMask(dfsClientConf.getUMask());
}
+ private FsPermission applyUMaskDir(FsPermission permission) {
+if (permission == null) {
+ permission = FsPermission.getDirDefault();
+}
+return permission.applyUMask(dfsClientConf.getUMask());
+ }
+
/**
* Same as {@link #create(String, FsPermission, EnumSet, boolean, short,
long,
* Progressable, int, ChecksumOpt)} with the addition of favoredNodes that is
@@ -2458,7 +2465,7 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
*
* @param src The path of the directory being created
* @param permission The permission of the directory being created.
- * If permission == null, use {@link FsPermission#getDefault()}.
+ * If permission == null, use {@link FsPermission#getDirDefault()}.
* @param createParent create missing parent directory if true
*
* @return True if the operation success.
@@ -2467,7 +2474,7 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
*/
public boolean mkdirs(String src, FsPermission permission,
boolean createParent) throws IOException {
-final FsPermission masked = applyUMask(permission);
+final FsPermission masked = applyUMaskDir(permission);
return primitiveMkdir(src, masked, createParent);
}
@@ -2488,9 +2495,8 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
boolean createParent) throws IOException {
checkOpen();
if (absPermission == null) {
- absPermission = applyUMask(null);
+ absPermission = applyUMaskDir(null);
}
-
LOG.debug("{}: masked={}", src, absPermission);
try (TraceScope ignored = tracer.newScope("mkdir")) {
return namenode.mkdirs(src, absPermission, createParent);
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermission.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermission.java
index 5e4f693..d3a4956 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermission.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermission.java
@@ -53,6 +53,7 @@ public class TestPermission {
final private static Path ROOT_PATH = new Path("/data");
final private static Path CHILD_DIR1 = new Path(ROOT_PATH, "child1");
final private static Path CHILD_DIR2 = new Path(ROOT_PATH, "child2");
+ final private static Path CHILD_DIR3 = new Path(ROOT_PATH, "child3");
final private static Path CHILD_FILE1 = new Path(ROOT_PATH, "file1");
final private static Path CHILD_FILE2 = new Path(ROOT_PATH, "file2");
final private static Path CHILD_FILE3 = new Path(ROOT_PATH, "file3");
@@ -237,6 +238,9 @@ public class TestPermission {
// following dir/file creations are legal
nnfs.mkdirs(CHILD_DIR1);
+ status = nnfs.getFileStatus(CHILD_DIR1);
+ assertThat("Expect 755 = 777 (default dir) - 022 (default umask)",
+ status.getPermission().toString(), is("rwxr-xr-x"));
out = nnfs.create(CHILD_FILE1);
status = nnfs.getFileStatus(CHILD_FILE1);
assertTrue(status.getPermission().toString().equals("rw-r--r--"));
@@ -248,6 +252,12 @@ public class TestPermission {
status = nnfs.getFileStatus(CHILD_FILE1);
assertTrue(status.getPermission().toString().equals("rwx--"));
+ // mkdirs with null permission
+ nnfs.mkdirs(CHILD_DIR3, null);
+ status = nnfs.getFileStatus(CHILD_DIR3);
+ assertThat("Expect 75
[hadoop] branch branch-3.3 updated: HDFS-15745. Make DataNodePeerMetrics#LOW_THRESHOLD_MS and MIN_OUTLIER_DETECTION_NODES configurable. Contributed by Haibin Huang. (#3992)
This is an automated email from the ASF dual-hosted git repository.
tasanuma pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 4c57fb4 HDFS-15745. Make DataNodePeerMetrics#LOW_THRESHOLD_MS and
MIN_OUTLIER_DETECTION_NODES configurable. Contributed by Haibin Huang. (#3992)
4c57fb4 is described below
commit 4c57fb4d6bb4cfe9b9269dd07b09270a969c34cf
Author: Takanobu Asanuma
AuthorDate: Wed Feb 16 09:42:43 2022 +0900
HDFS-15745. Make DataNodePeerMetrics#LOW_THRESHOLD_MS and
MIN_OUTLIER_DETECTION_NODES configurable. Contributed by Haibin Huang. (#3992)
(cherry picked from commit 1cd96e8dd81b220cc197011e80cd8e298c566e1a)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
Reviewed-by: Ayush Saxena
---
.../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 8
.../datanode/metrics/DataNodePeerMetrics.java | 24 +++---
.../src/main/resources/hdfs-default.xml| 16 +++
3 files changed, 41 insertions(+), 7 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 7196def..dc2bcbe 100755
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -650,6 +650,14 @@ public class DFSConfigKeys extends CommonConfigurationKeys
{
public static final long
DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_DEFAULT =
1000;
+ public static final String DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY =
+ "dfs.datanode.min.outlier.detection.nodes";
+ public static final long DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_DEFAULT =
+ 10L;
+ public static final String DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_KEY =
+ "dfs.datanode.slowpeer.low.threshold.ms";
+ public static final long DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_DEFAULT =
+ 5L;
public static final String DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY =
"dfs.datanode.min.outlier.detection.disks";
public static final long DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_DEFAULT =
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodePeerMetrics.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodePeerMetrics.java
index 3c70a23..750e53d 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodePeerMetrics.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodePeerMetrics.java
@@ -22,6 +22,7 @@ package org.apache.hadoop.hdfs.server.datanode.metrics;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.metrics2.MetricsJsonBuilder;
import org.apache.hadoop.metrics2.lib.MutableRollingAverages;
import org.slf4j.Logger;
@@ -48,11 +49,6 @@ public class DataNodePeerMetrics {
private final String name;
- /**
- * Threshold in milliseconds below which a DataNode is definitely not slow.
- */
- private static final long LOW_THRESHOLD_MS = 5;
- private static final long MIN_OUTLIER_DETECTION_NODES = 10;
private final OutlierDetector slowNodeDetector;
@@ -62,14 +58,28 @@ public class DataNodePeerMetrics {
* outlier detection is skipped.
*/
private final long minOutlierDetectionSamples;
+ /**
+ * Threshold in milliseconds below which a DataNode is definitely not slow.
+ */
+ private final long lowThresholdMs;
+ /**
+ * Minimum number of nodes to run outlier detection.
+ */
+ private final long minOutlierDetectionNodes;
public DataNodePeerMetrics(final String name, Configuration conf) {
this.name = name;
minOutlierDetectionSamples = conf.getLong(
DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_KEY,
DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_DEFAULT);
-this.slowNodeDetector = new OutlierDetector(MIN_OUTLIER_DETECTION_NODES,
-LOW_THRESHOLD_MS);
+lowThresholdMs =
+conf.getLong(DFSConfigKeys.DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_KEY,
+DFSConfigKeys.DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_DEFAULT);
+minOutlierDetectionNodes =
+
conf.getLong(DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY,
+DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_DEFAULT);
+this.slowNodeDetector =
+new OutlierDetector(minOutl
[hadoop] branch trunk updated: HADOOP-18109. Ensure that default permissions of directories under internal ViewFS directories are the same as directories on target filesystems. Contributed by Chentao
This is an automated email from the ASF dual-hosted git repository.
shv pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 19d90e6 HADOOP-18109. Ensure that default permissions of directories
under internal ViewFS directories are the same as directories on target
filesystems. Contributed by Chentao Yu. (3953)
19d90e6 is described below
commit 19d90e62fb28539f8c79bbb24f703301489825a6
Author: Chentao Yu
AuthorDate: Thu Apr 15 17:46:40 2021 -0700
HADOOP-18109. Ensure that default permissions of directories under internal
ViewFS directories are the same as directories on target filesystems.
Contributed by Chentao Yu. (3953)
---
.../org/apache/hadoop/fs/viewfs/ViewFileSystem.java | 5 -
.../hadoop/fs/viewfs/TestViewFileSystemHdfs.java | 19 +++
2 files changed, 19 insertions(+), 5 deletions(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index 5ff3c2b..8c3cdb8 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -1666,11 +1666,6 @@ public class ViewFileSystem extends FileSystem {
}
@Override
-public boolean mkdirs(Path dir) throws IOException {
- return mkdirs(dir, null);
-}
-
-@Override
public FSDataInputStream open(Path f, int bufferSize)
throws AccessControlException, FileNotFoundException, IOException {
checkPathIsSlash(f);
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
index fcb5257..fdc7464 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
@@ -479,4 +479,23 @@ public class TestViewFileSystemHdfs extends
ViewFileSystemBaseTest {
assertEquals("The owner did not match ", owner,
userUgi.getShortUserName());
otherfs.delete(user1Path, false);
}
+
+ @Test
+ public void testInternalDirectoryPermissions() throws IOException {
+LOG.info("Starting testInternalDirectoryPermissions!");
+Configuration localConf = new Configuration(conf);
+ConfigUtil.addLinkFallback(
+localConf, new Path(targetTestRoot, "fallbackDir").toUri());
+FileSystem fs = FileSystem.get(FsConstants.VIEWFS_URI, localConf);
+// check that the default permissions on a sub-folder of an internal
+// directory are the same as those created on non-internal directories.
+Path subDirOfInternalDir = new Path("/internalDir/dir1");
+fs.mkdirs(subDirOfInternalDir);
+
+Path subDirOfRealDir = new Path("/internalDir/linkToDir2/dir1");
+fs.mkdirs(subDirOfRealDir);
+
+assertEquals(fs.getFileStatus(subDirOfInternalDir).getPermission(),
+fs.getFileStatus(subDirOfRealDir).getPermission());
+ }
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.3 updated: HDFS-16396. Reconfig slow peer parameters for datanode (#3827)
This is an automated email from the ASF dual-hosted git repository.
tasanuma pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new db67952 HDFS-16396. Reconfig slow peer parameters for datanode (#3827)
db67952 is described below
commit db67952f9f38305ca263c033ed13fe772f79f300
Author: litao
AuthorDate: Tue Feb 15 12:40:46 2022 +0800
HDFS-16396. Reconfig slow peer parameters for datanode (#3827)
Reviewed-by: Ayush Saxena
(cherry picked from commit 0c194f2157ff4473ea95dff1d7d40c386398f4a4)
---
.../hdfs/server/datanode/BPServiceActor.java | 2 +-
.../hadoop/hdfs/server/datanode/BlockReceiver.java | 4 +-
.../apache/hadoop/hdfs/server/datanode/DNConf.java | 6 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 70 -
.../hadoop/hdfs/server/datanode/DataXceiver.java | 2 +-
.../datanode/metrics/DataNodePeerMetrics.java | 57 +++---
.../server/datanode/metrics/OutlierDetector.java | 20 -
.../datanode/TestDataNodeReconfiguration.java | 87 ++
.../org/apache/hadoop/hdfs/tools/TestDFSAdmin.java | 2 +-
9 files changed, 230 insertions(+), 20 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index 8f0286c..c8f18fa 100755
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -534,7 +534,7 @@ class BPServiceActor implements Runnable {
volumeFailureSummary.getFailedStorageLocations().length : 0;
final boolean outliersReportDue = scheduler.isOutliersReportDue(now);
final SlowPeerReports slowPeers =
-outliersReportDue && dn.getPeerMetrics() != null ?
+outliersReportDue && dnConf.peerStatsEnabled && dn.getPeerMetrics() !=
null ?
SlowPeerReports.create(dn.getPeerMetrics().getOutliers()) :
SlowPeerReports.EMPTY_REPORT;
final SlowDiskReports slowDisks =
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
index 315914a..e20f437 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
@@ -878,7 +878,7 @@ class BlockReceiver implements Closeable {
*/
private void trackSendPacketToLastNodeInPipeline(final long elapsedMs) {
final DataNodePeerMetrics peerMetrics = datanode.getPeerMetrics();
-if (peerMetrics != null && isPenultimateNode) {
+if (datanode.getDnConf().peerStatsEnabled && peerMetrics != null &&
isPenultimateNode) {
peerMetrics.addSendPacketDownstream(mirrorNameForMetrics, elapsedMs);
}
}
@@ -1093,7 +1093,7 @@ class BlockReceiver implements Closeable {
if (downstreams != null && downstreams.length > 0) {
downstreamDNs = downstreams;
isPenultimateNode = (downstreams.length == 1);
- if (isPenultimateNode && datanode.getPeerMetrics() != null) {
+ if (isPenultimateNode && datanode.getDnConf().peerStatsEnabled) {
mirrorNameForMetrics = (downstreams[0].getInfoSecurePort() != 0 ?
downstreams[0].getInfoSecureAddr() : downstreams[0].getInfoAddr());
LOG.debug("Will collect peer metrics for downstream node {}",
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
index 40d0df3..563fbde 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
@@ -108,7 +108,7 @@ public class DNConf {
private final long lifelineIntervalMs;
volatile long blockReportInterval;
volatile long blockReportSplitThreshold;
- final boolean peerStatsEnabled;
+ volatile boolean peerStatsEnabled;
final boolean diskStatsEnabled;
final long outliersReportIntervalMs;
final long ibrInterval;
@@ -507,4 +507,8 @@ public class DNConf {
dn.getConf().set(DFS_BLOCKREPORT_INITIAL_DELAY_KEY, delayMs);
initBlockReportDelay();
}
+
+ void setPeerStatsEnabled(boolean enablePeerStats) {
+peerStatsEnabled = enablePeerStats;
+ }
}
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
b/had
[hadoop] branch branch-3.3 updated: HADOOP-18109. Ensure that default permissions of directories under internal ViewFS directories are the same as directories on target filesystems. Contributed by Che
This is an automated email from the ASF dual-hosted git repository.
shv pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new d14a7c6 HADOOP-18109. Ensure that default permissions of directories
under internal ViewFS directories are the same as directories on target
filesystems. Contributed by Chentao Yu. (3953)
d14a7c6 is described below
commit d14a7c6ee5e881235d18d4cf9241197a1817d745
Author: Chentao Yu
AuthorDate: Thu Apr 15 17:46:40 2021 -0700
HADOOP-18109. Ensure that default permissions of directories under internal
ViewFS directories are the same as directories on target filesystems.
Contributed by Chentao Yu. (3953)
(cherry picked from commit 19d90e62fb28539f8c79bbb24f703301489825a6)
---
.../org/apache/hadoop/fs/viewfs/ViewFileSystem.java | 5 -
.../hadoop/fs/viewfs/TestViewFileSystemHdfs.java | 19 +++
2 files changed, 19 insertions(+), 5 deletions(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index 7503edd..8f333d1 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -1580,11 +1580,6 @@ public class ViewFileSystem extends FileSystem {
}
@Override
-public boolean mkdirs(Path dir) throws IOException {
- return mkdirs(dir, null);
-}
-
-@Override
public FSDataInputStream open(Path f, int bufferSize)
throws AccessControlException, FileNotFoundException, IOException {
checkPathIsSlash(f);
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
index fcb5257..fdc7464 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
@@ -479,4 +479,23 @@ public class TestViewFileSystemHdfs extends
ViewFileSystemBaseTest {
assertEquals("The owner did not match ", owner,
userUgi.getShortUserName());
otherfs.delete(user1Path, false);
}
+
+ @Test
+ public void testInternalDirectoryPermissions() throws IOException {
+LOG.info("Starting testInternalDirectoryPermissions!");
+Configuration localConf = new Configuration(conf);
+ConfigUtil.addLinkFallback(
+localConf, new Path(targetTestRoot, "fallbackDir").toUri());
+FileSystem fs = FileSystem.get(FsConstants.VIEWFS_URI, localConf);
+// check that the default permissions on a sub-folder of an internal
+// directory are the same as those created on non-internal directories.
+Path subDirOfInternalDir = new Path("/internalDir/dir1");
+fs.mkdirs(subDirOfInternalDir);
+
+Path subDirOfRealDir = new Path("/internalDir/linkToDir2/dir1");
+fs.mkdirs(subDirOfRealDir);
+
+assertEquals(fs.getFileStatus(subDirOfInternalDir).getPermission(),
+fs.getFileStatus(subDirOfRealDir).getPermission());
+ }
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.2 updated: HADOOP-18109. Ensure that default permissions of directories under internal ViewFS directories are the same as directories on target filesystems. Contributed by Che
This is an automated email from the ASF dual-hosted git repository.
shv pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 5066722 HADOOP-18109. Ensure that default permissions of directories
under internal ViewFS directories are the same as directories on target
filesystems. Contributed by Chentao Yu. (3953)
5066722 is described below
commit 5066722eb70b82a532b55ce98a2623852fd195c8
Author: Chentao Yu
AuthorDate: Thu Apr 15 17:46:40 2021 -0700
HADOOP-18109. Ensure that default permissions of directories under internal
ViewFS directories are the same as directories on target filesystems.
Contributed by Chentao Yu. (3953)
(cherry picked from commit 19d90e62fb28539f8c79bbb24f703301489825a6)
---
.../org/apache/hadoop/fs/viewfs/ViewFileSystem.java | 5 -
.../hadoop/fs/viewfs/TestViewFileSystemHdfs.java | 19 +++
2 files changed, 19 insertions(+), 5 deletions(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index a430727..0ff2e73 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -1496,11 +1496,6 @@ public class ViewFileSystem extends FileSystem {
}
@Override
-public boolean mkdirs(Path dir) throws IOException {
- return mkdirs(dir, null);
-}
-
-@Override
public FSDataInputStream open(Path f, int bufferSize)
throws AccessControlException, FileNotFoundException, IOException {
checkPathIsSlash(f);
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
index fcb5257..fdc7464 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
@@ -479,4 +479,23 @@ public class TestViewFileSystemHdfs extends
ViewFileSystemBaseTest {
assertEquals("The owner did not match ", owner,
userUgi.getShortUserName());
otherfs.delete(user1Path, false);
}
+
+ @Test
+ public void testInternalDirectoryPermissions() throws IOException {
+LOG.info("Starting testInternalDirectoryPermissions!");
+Configuration localConf = new Configuration(conf);
+ConfigUtil.addLinkFallback(
+localConf, new Path(targetTestRoot, "fallbackDir").toUri());
+FileSystem fs = FileSystem.get(FsConstants.VIEWFS_URI, localConf);
+// check that the default permissions on a sub-folder of an internal
+// directory are the same as those created on non-internal directories.
+Path subDirOfInternalDir = new Path("/internalDir/dir1");
+fs.mkdirs(subDirOfInternalDir);
+
+Path subDirOfRealDir = new Path("/internalDir/linkToDir2/dir1");
+fs.mkdirs(subDirOfRealDir);
+
+assertEquals(fs.getFileStatus(subDirOfInternalDir).getPermission(),
+fs.getFileStatus(subDirOfRealDir).getPermission());
+ }
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
