YARN-5718. TimelineClient (and other places in YARN) shouldn't over-write HDFS client retry settings which could cause unexpected behavior. Contributed by Junping Du.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b733a6f8 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b733a6f8 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b733a6f8 Branch: refs/heads/HADOOP-13070 Commit: b733a6f86262522e535cebc972baecbe6a6eab50 Parents: b154d3e Author: Xuan <xg...@apache.org> Authored: Tue Oct 18 11:04:49 2016 -0700 Committer: Xuan <xg...@apache.org> Committed: Tue Oct 18 11:06:47 2016 -0700 ---------------------------------------------------------------------- .../hadoop/yarn/conf/YarnConfiguration.java | 17 +---------------- .../api/impl/FileSystemTimelineWriter.java | 7 ------- .../nodelabels/FileSystemNodeLabelsStore.java | 7 +------ .../src/main/resources/yarn-default.xml | 20 -------------------- .../recovery/FileSystemRMStateStore.java | 5 ----- .../recovery/TestFSRMStateStore.java | 4 ---- 6 files changed, 2 insertions(+), 58 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b733a6f8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 3bd0dcc..1a30c32 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -695,10 +695,6 @@ public class YarnConfiguration extends Configuration { /** URI for FileSystemRMStateStore */ public static final String FS_RM_STATE_STORE_URI = RM_PREFIX + "fs.state-store.uri"; - public static final String FS_RM_STATE_STORE_RETRY_POLICY_SPEC = RM_PREFIX - + "fs.state-store.retry-policy-spec"; - public static final String DEFAULT_FS_RM_STATE_STORE_RETRY_POLICY_SPEC = - "2000, 500"; public static final String FS_RM_STATE_STORE_NUM_RETRIES = RM_PREFIX + "fs.state-store.num-retries"; @@ -1974,13 +1970,6 @@ public class YarnConfiguration extends Configuration { TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_UNKNOWN_ACTIVE_SECONDS_DEFAULT = 24 * 60 * 60; - public static final String - TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RETRY_POLICY_SPEC = - TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX + "retry-policy-spec"; - public static final String - DEFAULT_TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RETRY_POLICY_SPEC = - "2000, 500"; - public static final String TIMELINE_SERVICE_LEVELDB_CACHE_READ_CACHE_SIZE = TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX + "leveldb-cache-read-cache-size"; @@ -2600,11 +2589,7 @@ public class YarnConfiguration extends Configuration { /** URI for NodeLabelManager */ public static final String FS_NODE_LABELS_STORE_ROOT_DIR = NODE_LABELS_PREFIX + "fs-store.root-dir"; - public static final String FS_NODE_LABELS_STORE_RETRY_POLICY_SPEC = - NODE_LABELS_PREFIX + "fs-store.retry-policy-spec"; - public static final String DEFAULT_FS_NODE_LABELS_STORE_RETRY_POLICY_SPEC = - "2000, 500"; - + /** * Flag to indicate if the node labels feature enabled, by default it's * disabled http://git-wip-us.apache.org/repos/asf/hadoop/blob/b733a6f8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java index 55d6bd2..b1284e1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java @@ -106,13 +106,6 @@ public class FileSystemTimelineWriter extends TimelineWriter{ super(authUgi, client, resURI); Configuration fsConf = new Configuration(conf); - fsConf.setBoolean("dfs.client.retry.policy.enabled", true); - String retryPolicy = - fsConf.get(YarnConfiguration. - TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RETRY_POLICY_SPEC, - YarnConfiguration. - DEFAULT_TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RETRY_POLICY_SPEC); - fsConf.set("dfs.client.retry.policy.spec", retryPolicy); activePath = new Path(fsConf.get( YarnConfiguration http://git-wip-us.apache.org/repos/asf/hadoop/blob/b733a6f8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java index a65349b..9a2b8be 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java @@ -95,13 +95,8 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore { void setFileSystem(Configuration conf) throws IOException { Configuration confCopy = new Configuration(conf); - confCopy.setBoolean("dfs.client.retry.policy.enabled", true); - String retryPolicy = - confCopy.get(YarnConfiguration.FS_NODE_LABELS_STORE_RETRY_POLICY_SPEC, - YarnConfiguration.DEFAULT_FS_NODE_LABELS_STORE_RETRY_POLICY_SPEC); - confCopy.set("dfs.client.retry.policy.spec", retryPolicy); fs = fsWorkingPath.getFileSystem(confCopy); - + // if it's local file system, use RawLocalFileSystem instead of // LocalFileSystem, the latter one doesn't support append. if (fs.getScheme().equals("file")) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/b733a6f8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index f37c689..72e026d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -522,16 +522,6 @@ </property> <property> - <description>hdfs client retry policy specification. hdfs client retry - is always enabled. Specified in pairs of sleep-time and number-of-retries - and (t0, n0), (t1, n1), ..., the first n0 retries sleep t0 milliseconds on - average, the following n1 retries sleep t1 milliseconds on average, and so on. - </description> - <name>yarn.resourcemanager.fs.state-store.retry-policy-spec</name> - <value>2000, 500</value> - </property> - - <property> <description>the number of retries to recover from IOException in FileSystemRMStateStore. </description> @@ -2483,16 +2473,6 @@ <property> <description> - Retry policy used for FileSystem node label store. The policy is - specified by N pairs of sleep-time in milliseconds and number-of-retries - "s1,n1,s2,n2,...". - </description> - <name>yarn.node-labels.fs-store.retry-policy-spec</name> - <value>2000, 500</value> - </property> - - <property> - <description> URI for NodeLabelManager. The default value is /tmp/hadoop-yarn-${user}/node-labels/ in the local filesystem. </description> http://git-wip-us.apache.org/repos/asf/hadoop/blob/b733a6f8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java index a228037..b9a4374 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java @@ -142,11 +142,6 @@ public class FileSystemRMStateStore extends RMStateStore { // authenticated with kerberos so we are good to create a file-system // handle. fsConf = new Configuration(getConfig()); - fsConf.setBoolean("dfs.client.retry.policy.enabled", true); - String retryPolicy = - fsConf.get(YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC, - YarnConfiguration.DEFAULT_FS_RM_STATE_STORE_RETRY_POLICY_SPEC); - fsConf.set("dfs.client.retry.policy.spec", retryPolicy); String scheme = fsWorkingPath.toUri().getScheme(); if (scheme == null) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/b733a6f8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java index 61088e1..6f0d53f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java @@ -113,8 +113,6 @@ public class TestFSRMStateStore extends RMStateStoreTestBase { YarnConfiguration conf = new YarnConfiguration(); conf.set(YarnConfiguration.FS_RM_STATE_STORE_URI, workingDirPathURI.toString()); - conf.set(YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC, - "100,6000"); conf.setInt(YarnConfiguration.FS_RM_STATE_STORE_NUM_RETRIES, 8); conf.setLong(YarnConfiguration.FS_RM_STATE_STORE_RETRY_INTERVAL_MS, 900L); @@ -325,8 +323,6 @@ public class TestFSRMStateStore extends RMStateStoreTestBase { YarnConfiguration conf = new YarnConfiguration(); conf.set(YarnConfiguration.FS_RM_STATE_STORE_URI, workingDirPathURI.toString()); - conf.set(YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC, - "100,6000"); this.store = new TestFileSystemRMStore(conf) { Version storedVersion = null; --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org