subject:"\[45\/50\] \[abbrv\] hadoop git commit\: HDFS\-13166\: \[SPS\]\: Implement caching mechanism to keep LIVE datanodes to minimize costly getLiveDatanodeStorageReport\(\) calls. Contributed by Rakesh R."

[45/50] [abbrv] hadoop git commit: HDFS-13166: [SPS]: Implement caching mechanism to keep LIVE datanodes to minimize costly getLiveDatanodeStorageReport() calls. Contributed by Rakesh R.

2018-07-19 Thread rakeshr

HDFS-13166: [SPS]: Implement caching mechanism to keep LIVE datanodes to 
minimize costly getLiveDatanodeStorageReport() calls. Contributed by Rakesh R.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6fed235a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6fed235a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6fed235a

Branch: refs/heads/HDFS-10285
Commit: 6fed235aeb0aa18b2e7b2d31f37fad9c368ecaeb
Parents: 6915639
Author: Surendra Singh Lilhore 
Authored: Thu Mar 1 00:08:37 2018 +0530
Committer: Rakesh Radhakrishnan 
Committed: Thu Jul 19 22:49:58 2018 +0530

--
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |   5 +
 .../NamenodeProtocolServerSideTranslatorPB.java |  19 --
 .../NamenodeProtocolTranslatorPB.java   |  17 -
 .../hdfs/server/namenode/NameNodeRpcServer.java |  13 -
 .../hdfs/server/namenode/sps/Context.java   |  24 +-
 .../namenode/sps/DatanodeCacheManager.java  | 121 +++
 .../namenode/sps/IntraSPSNameNodeContext.java   |  23 +-
 .../namenode/sps/StoragePolicySatisfier.java| 340 ++-
 .../hdfs/server/protocol/NamenodeProtocol.java  |  16 -
 .../hdfs/server/sps/ExternalSPSContext.java |  32 +-
 .../src/main/proto/NamenodeProtocol.proto   |  25 --
 .../src/main/resources/hdfs-default.xml |  11 +
 .../src/site/markdown/ArchivalStorage.md|   2 +-
 .../TestStoragePolicySatisfyWorker.java |   3 +
 .../TestPersistentStoragePolicySatisfier.java   |   6 +
 .../TestStoragePolicySatisfierWithHA.java   |   3 +
 .../sps/TestStoragePolicySatisfier.java |   4 +
 ...stStoragePolicySatisfierWithStripedFile.java |  24 +-
 .../TestStoragePolicySatisfyAdminCommands.java  |   3 +
 19 files changed, 431 insertions(+), 260 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fed235a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index c79bf60..843d9d8 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -640,6 +640,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys 
{
   public static final String DFS_SPS_MAX_OUTSTANDING_PATHS_KEY =
   "dfs.storage.policy.satisfier.max.outstanding.paths";
   public static final int DFS_SPS_MAX_OUTSTANDING_PATHS_DEFAULT = 1;
+  // SPS datanode cache config, defaulting to 5mins.
+  public static final String DFS_SPS_DATANODE_CACHE_REFRESH_INTERVAL_MS =
+  "dfs.storage.policy.satisfier.datanode.cache.refresh.interval.ms";
+  public static final long DFS_SPS_DATANODE_CACHE_REFRESH_INTERVAL_MS_DEFAULT =
+  30L;
 
   // SPS keytab configurations, by default it is disabled.
   public static final String  DFS_SPS_ADDRESS_KEY =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fed235a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
index ed176cc..e4283c6 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
@@ -23,8 +23,6 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo.DatanodeInfoBuilder;
 import 
org.apache.hadoop.hdfs.protocol.proto.HdfsServerProtos.VersionRequestProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.HdfsServerProtos.VersionResponseProto;
-import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.CheckDNSpaceRequestProto;
-import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.CheckDNSpaceResponseProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.EndCheckpointRequestProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.EndCheckpointResponseProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.ErrorReportRequestProto;
@@ -277,21 +275,4 @@ public class NamenodeProtocolServerSideTranslatorPB

[45/50] [abbrv] hadoop git commit: HDFS-13166: [SPS]: Implement caching mechanism to keep LIVE datanodes to minimize costly getLiveDatanodeStorageReport() calls. Contributed by Rakesh R.

2018-07-16 Thread rakeshr

HDFS-13166: [SPS]: Implement caching mechanism to keep LIVE datanodes to 
minimize costly getLiveDatanodeStorageReport() calls. Contributed by Rakesh R.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96fe124b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96fe124b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96fe124b

Branch: refs/heads/HDFS-10285
Commit: 96fe124ba8654aa725150455d7ca790187ff10d6
Parents: 9dca6ea
Author: Surendra Singh Lilhore 
Authored: Thu Mar 1 00:08:37 2018 +0530
Committer: Rakesh Radhakrishnan 
Committed: Sun Jul 15 20:23:23 2018 +0530

--
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |   5 +
 .../NamenodeProtocolServerSideTranslatorPB.java |  19 --
 .../NamenodeProtocolTranslatorPB.java   |  17 -
 .../hdfs/server/namenode/NameNodeRpcServer.java |  13 -
 .../hdfs/server/namenode/sps/Context.java   |  24 +-
 .../namenode/sps/DatanodeCacheManager.java  | 121 +++
 .../namenode/sps/IntraSPSNameNodeContext.java   |  23 +-
 .../namenode/sps/StoragePolicySatisfier.java| 340 ++-
 .../hdfs/server/protocol/NamenodeProtocol.java  |  16 -
 .../hdfs/server/sps/ExternalSPSContext.java |  32 +-
 .../src/main/proto/NamenodeProtocol.proto   |  25 --
 .../src/main/resources/hdfs-default.xml |  11 +
 .../src/site/markdown/ArchivalStorage.md|   2 +-
 .../TestStoragePolicySatisfyWorker.java |   3 +
 .../TestPersistentStoragePolicySatisfier.java   |   6 +
 .../TestStoragePolicySatisfierWithHA.java   |   3 +
 .../sps/TestStoragePolicySatisfier.java |   4 +
 ...stStoragePolicySatisfierWithStripedFile.java |  24 +-
 .../TestStoragePolicySatisfyAdminCommands.java  |   3 +
 19 files changed, 431 insertions(+), 260 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/96fe124b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index c79bf60..843d9d8 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -640,6 +640,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys 
{
   public static final String DFS_SPS_MAX_OUTSTANDING_PATHS_KEY =
   "dfs.storage.policy.satisfier.max.outstanding.paths";
   public static final int DFS_SPS_MAX_OUTSTANDING_PATHS_DEFAULT = 1;
+  // SPS datanode cache config, defaulting to 5mins.
+  public static final String DFS_SPS_DATANODE_CACHE_REFRESH_INTERVAL_MS =
+  "dfs.storage.policy.satisfier.datanode.cache.refresh.interval.ms";
+  public static final long DFS_SPS_DATANODE_CACHE_REFRESH_INTERVAL_MS_DEFAULT =
+  30L;
 
   // SPS keytab configurations, by default it is disabled.
   public static final String  DFS_SPS_ADDRESS_KEY =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96fe124b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
index ed176cc..e4283c6 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
@@ -23,8 +23,6 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo.DatanodeInfoBuilder;
 import 
org.apache.hadoop.hdfs.protocol.proto.HdfsServerProtos.VersionRequestProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.HdfsServerProtos.VersionResponseProto;
-import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.CheckDNSpaceRequestProto;
-import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.CheckDNSpaceResponseProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.EndCheckpointRequestProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.EndCheckpointResponseProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.ErrorReportRequestProto;
@@ -277,21 +275,4 @@ public class NamenodeProtocolServerSideTranslatorPB

[45/50] [abbrv] hadoop git commit: HDFS-13166: [SPS]: Implement caching mechanism to keep LIVE datanodes to minimize costly getLiveDatanodeStorageReport() calls. Contributed by Rakesh R.

2018-07-12 Thread rakeshr

HDFS-13166: [SPS]: Implement caching mechanism to keep LIVE datanodes to 
minimize costly getLiveDatanodeStorageReport() calls. Contributed by Rakesh R.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dc52bd4b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dc52bd4b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dc52bd4b

Branch: refs/heads/HDFS-10285
Commit: dc52bd4bdfde6d5153bd30b34f60ae82bfb6379e
Parents: 538b9bd
Author: Surendra Singh Lilhore 
Authored: Thu Mar 1 00:08:37 2018 +0530
Committer: Rakesh Radhakrishnan 
Committed: Thu Jul 12 17:04:39 2018 +0530

--
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |   5 +
 .../NamenodeProtocolServerSideTranslatorPB.java |  19 --
 .../NamenodeProtocolTranslatorPB.java   |  17 -
 .../hdfs/server/namenode/NameNodeRpcServer.java |  13 -
 .../hdfs/server/namenode/sps/Context.java   |  24 +-
 .../namenode/sps/DatanodeCacheManager.java  | 121 +++
 .../namenode/sps/IntraSPSNameNodeContext.java   |  23 +-
 .../namenode/sps/StoragePolicySatisfier.java| 340 ++-
 .../hdfs/server/protocol/NamenodeProtocol.java  |  16 -
 .../hdfs/server/sps/ExternalSPSContext.java |  32 +-
 .../src/main/proto/NamenodeProtocol.proto   |  25 --
 .../src/main/resources/hdfs-default.xml |  11 +
 .../src/site/markdown/ArchivalStorage.md|   2 +-
 .../TestStoragePolicySatisfyWorker.java |   3 +
 .../TestPersistentStoragePolicySatisfier.java   |   6 +
 .../TestStoragePolicySatisfierWithHA.java   |   3 +
 .../sps/TestStoragePolicySatisfier.java |   4 +
 ...stStoragePolicySatisfierWithStripedFile.java |  24 +-
 .../TestStoragePolicySatisfyAdminCommands.java  |   3 +
 19 files changed, 431 insertions(+), 260 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc52bd4b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index c79bf60..843d9d8 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -640,6 +640,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys 
{
   public static final String DFS_SPS_MAX_OUTSTANDING_PATHS_KEY =
   "dfs.storage.policy.satisfier.max.outstanding.paths";
   public static final int DFS_SPS_MAX_OUTSTANDING_PATHS_DEFAULT = 1;
+  // SPS datanode cache config, defaulting to 5mins.
+  public static final String DFS_SPS_DATANODE_CACHE_REFRESH_INTERVAL_MS =
+  "dfs.storage.policy.satisfier.datanode.cache.refresh.interval.ms";
+  public static final long DFS_SPS_DATANODE_CACHE_REFRESH_INTERVAL_MS_DEFAULT =
+  30L;
 
   // SPS keytab configurations, by default it is disabled.
   public static final String  DFS_SPS_ADDRESS_KEY =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc52bd4b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
index ed176cc..e4283c6 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
@@ -23,8 +23,6 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo.DatanodeInfoBuilder;
 import 
org.apache.hadoop.hdfs.protocol.proto.HdfsServerProtos.VersionRequestProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.HdfsServerProtos.VersionResponseProto;
-import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.CheckDNSpaceRequestProto;
-import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.CheckDNSpaceResponseProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.EndCheckpointRequestProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.EndCheckpointResponseProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.ErrorReportRequestProto;
@@ -277,21 +275,4 @@ public class NamenodeProtocolServerSideTranslatorPB

[45/50] [abbrv] hadoop git commit: HDFS-13166: [SPS]: Implement caching mechanism to keep LIVE datanodes to minimize costly getLiveDatanodeStorageReport() calls. Contributed by Rakesh R.

[45/50] [abbrv] hadoop git commit: HDFS-13166: [SPS]: Implement caching mechanism to keep LIVE datanodes to minimize costly getLiveDatanodeStorageReport() calls. Contributed by Rakesh R.

[45/50] [abbrv] hadoop git commit: HDFS-13166: [SPS]: Implement caching mechanism to keep LIVE datanodes to minimize costly getLiveDatanodeStorageReport() calls. Contributed by Rakesh R.

3 matches

Site Navigation

Mail list logo

Footer information