Repository: hadoop Updated Branches: refs/heads/branch-3.0 4b1a2152e -> ee4525e07
HDFS-11754. Make FsServerDefaults cache configurable. Contributed by Mikhail Erofeev. (cherry picked from commit 53509f295b5274059541565d7216bf98aa35347d) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee4525e0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee4525e0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee4525e0 Branch: refs/heads/branch-3.0 Commit: ee4525e071377c8d56b3673dc6a61b8acd478976 Parents: 4b1a215 Author: Kihwal Lee <kih...@apache.org> Authored: Wed Nov 29 15:12:47 2017 -0600 Committer: Kihwal Lee <kih...@apache.org> Committed: Wed Nov 29 15:12:47 2017 -0600 ---------------------------------------------------------------------- .../java/org/apache/hadoop/hdfs/DFSClient.java | 10 +- .../hdfs/client/HdfsClientConfigKeys.java | 4 + .../src/main/resources/hdfs-default.xml | 10 ++ .../apache/hadoop/hdfs/TestFileCreation.java | 103 +++++++++++++++++++ 4 files changed, 124 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee4525e0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 5652321..25e0f6c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -24,6 +24,8 @@ import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_CACH import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_CONTEXT; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_CONTEXT_DEFAULT; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_LOCAL_INTERFACES; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS_DEFAULT; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS_KEY; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_DEFAULT; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY; @@ -205,8 +207,6 @@ import com.google.common.net.InetAddresses; public class DFSClient implements java.io.Closeable, RemotePeerFactory, DataEncryptionKeyFactory { public static final Logger LOG = LoggerFactory.getLogger(DFSClient.class); - // 1 hour - public static final long SERVER_DEFAULTS_VALIDITY_PERIOD = 60 * 60 * 1000L; private static final String DFS_KMS_PREFIX = "dfs-kms-"; private final Configuration conf; @@ -240,6 +240,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, private static ThreadPoolExecutor HEDGED_READ_THREAD_POOL; private static volatile ThreadPoolExecutor STRIPED_READ_THREAD_POOL; private final int smallBufferSize; + private final long serverDefaultsValidityPeriod; public DfsClientConf getConf() { return dfsClientConf; @@ -371,6 +372,9 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, null : conf.getBoolean(DFS_CLIENT_CACHE_DROP_BEHIND_READS, false); Long readahead = (conf.get(DFS_CLIENT_CACHE_READAHEAD) == null) ? null : conf.getLong(DFS_CLIENT_CACHE_READAHEAD, 0); + this.serverDefaultsValidityPeriod = + conf.getLong(DFS_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS_KEY, + DFS_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS_DEFAULT); Boolean writeDropBehind = (conf.get(DFS_CLIENT_CACHE_DROP_BEHIND_WRITES) == null) ? null : conf.getBoolean(DFS_CLIENT_CACHE_DROP_BEHIND_WRITES, false); @@ -663,7 +667,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, checkOpen(); long now = Time.monotonicNow(); if ((serverDefaults == null) || - (now - serverDefaultsLastUpdate > SERVER_DEFAULTS_VALIDITY_PERIOD)) { + (now - serverDefaultsLastUpdate > serverDefaultsValidityPeriod)) { serverDefaults = namenode.getServerDefaults(); serverDefaultsLastUpdate = now; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee4525e0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java index 97cb68b..d6c47b5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java @@ -112,6 +112,10 @@ public interface HdfsClientConfigKeys { String DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_KEY = "dfs.client.max.block.acquire.failures"; int DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_DEFAULT = 3; + String DFS_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS_KEY = + "dfs.client.server-defaults.validity.period.ms"; + long DFS_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS_DEFAULT = + TimeUnit.HOURS.toMillis(1); String DFS_CHECKSUM_TYPE_KEY = "dfs.checksum.type"; String DFS_CHECKSUM_TYPE_DEFAULT = "CRC32C"; String DFS_BYTES_PER_CHECKSUM_KEY = "dfs.bytes-per-checksum"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee4525e0/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 158786c..c5f3582 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -2283,6 +2283,16 @@ </property> <property> + <name>dfs.client.server-defaults.validity.period.ms</name> + <value>3600000</value> + <description> + The amount of milliseconds after which cached server defaults are updated. + + By default this parameter is set to 1 hour. + </description> +</property> + +<property> <name>dfs.namenode.enable.retrycache</name> <value>true</value> <description> http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee4525e0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java index e191414..13cc288 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java @@ -23,6 +23,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS_KEY; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SYNCONCLOSE_KEY; @@ -38,6 +39,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.junit.Assume.assumeTrue; +import static org.mockito.Mockito.doReturn; import java.io.BufferedReader; import java.io.ByteArrayOutputStream; @@ -51,6 +53,7 @@ import java.net.URI; import java.net.UnknownHostException; import java.security.PrivilegedExceptionAction; import java.util.EnumSet; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; @@ -179,6 +182,106 @@ public class TestFileCreation { } } + /** + * Test that server default values are cached on the client size + * and are stale after namenode update. + */ + @Test + public void testServerDefaultsWithCaching() + throws IOException, InterruptedException { + // Create cluster with an explicit block size param + Configuration clusterConf = new HdfsConfiguration(); + long originalBlockSize = DFS_BLOCK_SIZE_DEFAULT * 2; + clusterConf.setLong(DFS_BLOCK_SIZE_KEY, originalBlockSize); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(clusterConf) + .numDataNodes(0) + .build(); + cluster.waitActive(); + // Set a spy namesystem inside the namenode and return it + FSNamesystem spyNamesystem = + NameNodeAdapter.spyOnNamesystem(cluster.getNameNode()); + InetSocketAddress nameNodeAddr = cluster.getNameNode().getNameNodeAddress(); + try { + // Create a dfs client and set a long enough validity interval + Configuration clientConf = new HdfsConfiguration(); + clientConf.setLong(DFS_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS_KEY, + TimeUnit.MINUTES.toMillis(1)); + DFSClient dfsClient = new DFSClient(nameNodeAddr, clientConf); + FsServerDefaults defaults = dfsClient.getServerDefaults(); + assertEquals(originalBlockSize, defaults.getBlockSize()); + + // Update the namenode with a new parameter + long updatedDefaultBlockSize = DFS_BLOCK_SIZE_DEFAULT * 3; + FsServerDefaults newDefaults = + new FsServerDefaults(updatedDefaultBlockSize, + defaults.getBytesPerChecksum(), defaults.getWritePacketSize(), + defaults.getReplication(), defaults.getFileBufferSize(), + defaults.getEncryptDataTransfer(), defaults.getTrashInterval(), + defaults.getChecksumType(), defaults.getKeyProviderUri(), + defaults.getDefaultStoragePolicyId()); + doReturn(newDefaults).when(spyNamesystem).getServerDefaults(); + + // The value is stale + Thread.sleep(1); + defaults = dfsClient.getServerDefaults(); + assertEquals(originalBlockSize, defaults.getBlockSize()); + + // Another client reads the updated value correctly + DFSClient newDfsClient = new DFSClient(nameNodeAddr, clientConf); + defaults = newDfsClient.getServerDefaults(); + assertEquals(updatedDefaultBlockSize, defaults.getBlockSize()); + } finally { + cluster.shutdown(); + } + } + + /** + * Test that server defaults are updated on the client after cache expiration. + */ + @Test + public void testServerDefaultsWithMinimalCaching() + throws IOException, InterruptedException { + // Create cluster with an explicit block size param + Configuration clusterConf = new HdfsConfiguration(); + long originalBlockSize = DFS_BLOCK_SIZE_DEFAULT * 2; + clusterConf.setLong(DFS_BLOCK_SIZE_KEY, originalBlockSize); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(clusterConf) + .numDataNodes(0) + .build(); + cluster.waitActive(); + // Set a spy namesystem inside the namenode and return it + FSNamesystem spyNamesystem = + NameNodeAdapter.spyOnNamesystem(cluster.getNameNode()); + InetSocketAddress nameNodeAddr = cluster.getNameNode().getNameNodeAddress(); + try { + // Create a dfs client and set a minimal validity interval + Configuration clientConf = new HdfsConfiguration(); + // Invalidate cache in at most 1 ms, see DfsClient#getServerDefaults + clientConf.setLong(DFS_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS_KEY, 0L); + DFSClient dfsClient = new DFSClient(nameNodeAddr, clientConf); + FsServerDefaults defaults = dfsClient.getServerDefaults(); + assertEquals(originalBlockSize, defaults.getBlockSize()); + + // Update the namenode with a new FsServerDefaults + long updatedDefaultBlockSize = DFS_BLOCK_SIZE_DEFAULT * 3; + FsServerDefaults newDefaults = + new FsServerDefaults(updatedDefaultBlockSize, + defaults.getBytesPerChecksum(), defaults.getWritePacketSize(), + defaults.getReplication(), defaults.getFileBufferSize(), + defaults.getEncryptDataTransfer(), defaults.getTrashInterval(), + defaults.getChecksumType(), defaults.getKeyProviderUri(), + defaults.getDefaultStoragePolicyId()); + doReturn(newDefaults).when(spyNamesystem).getServerDefaults(); + + Thread.sleep(1); + defaults = dfsClient.getServerDefaults(); + // Value is updated correctly + assertEquals(updatedDefaultBlockSize, defaults.getBlockSize()); + } finally { + cluster.shutdown(); + } + } + @Test public void testFileCreation() throws IOException { checkFileCreation(null, false); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org