This is an automated email from the ASF dual-hosted git repository. sunilg pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push: new fb1ecff Revert "YARN-9873. Mutation API Config Change updates Version Number. Contributed by Prabhu Joseph" fb1ecff is described below commit fb1ecff6a26875c7f2b86ef07d7b9145c469377e Author: Sunil G <sun...@apache.org> AuthorDate: Sat Oct 5 09:15:17 2019 +0530 Revert "YARN-9873. Mutation API Config Change updates Version Number. Contributed by Prabhu Joseph" This reverts commit 4510970e2f7728d036c750b596985e5ffa357b60. --- .../scheduler/MutableConfigurationProvider.java | 6 ---- .../conf/FSSchedulerConfigurationStore.java | 7 ----- .../capacity/conf/InMemoryConfigurationStore.java | 8 ----- .../capacity/conf/LeveldbConfigurationStore.java | 23 +-------------- .../conf/MutableCSConfigurationProvider.java | 5 ---- .../capacity/conf/YarnConfigurationStore.java | 6 ---- .../capacity/conf/ZKConfigurationStore.java | 19 ------------ .../server/resourcemanager/webapp/RMWSConsts.java | 3 -- .../resourcemanager/webapp/RMWebServices.java | 34 +--------------------- .../capacity/conf/TestZKConfigurationStore.java | 15 ---------- 10 files changed, 2 insertions(+), 124 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java index eff8aa8..9e843df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java @@ -65,12 +65,6 @@ public interface MutableConfigurationProvider { */ Configuration getConfiguration(); - /** - * Get the last updated scheduler config version. - * @return Last updated scheduler config version. - */ - long getConfigVersion() throws Exception; - void formatConfigurationInStore(Configuration conf) throws Exception; /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java index f59fa0a..80053be 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java @@ -148,13 +148,6 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore { tempConfigPath = null; } - @Override - public long getConfigVersion() throws Exception { - String version = getLatestConfigPath().getName(). - substring(YarnConfiguration.CS_CONFIGURATION_FILE.length() + 1); - return Long.parseLong(version); - } - private void finalizeFileSystemFile() throws IOException { // call confirmMutation() make sure tempConfigPath is not null Path finalConfigPath = getFinalConfigPath(tempConfigPath); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java index 59d140e..4871443 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java @@ -33,13 +33,11 @@ public class InMemoryConfigurationStore extends YarnConfigurationStore { private Configuration schedConf; private LogMutation pendingMutation; - private long configVersion; @Override public void initialize(Configuration conf, Configuration schedConf, RMContext rmContext) { this.schedConf = schedConf; - this.configVersion = System.currentTimeMillis(); } @Override @@ -59,7 +57,6 @@ public class InMemoryConfigurationStore extends YarnConfigurationStore { } } } - this.configVersion = System.currentTimeMillis(); pendingMutation = null; } @@ -74,11 +71,6 @@ public class InMemoryConfigurationStore extends YarnConfigurationStore { } @Override - public long getConfigVersion() throws Exception { - return configVersion; - } - - @Override public List<LogMutation> getConfirmedConfHistory(long fromId) { // Unimplemented. return null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java index 2ae4f05..743d7ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java @@ -68,7 +68,6 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore { private static final String DB_NAME = "yarn-conf-store"; private static final String LOG_KEY = "log"; private static final String VERSION_KEY = "version"; - private static final String CONF_VERSION_KEY = "conf-version"; private DB db; private long maxLogs; @@ -125,10 +124,6 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore { return 1; } else if (key2Str.equals(LOG_KEY)) { return -1; - } else if (key1Str.equals(CONF_VERSION_KEY)) { - return 1; - } else if (key2Str.equals(CONF_VERSION_KEY)) { - return -1; } return key1Str.compareTo(key2Str); } @@ -151,10 +146,6 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore { File dbfile = new File(storeRoot.toString()); try { db = JniDBFactory.factory.open(dbfile, options); - if (db.get(bytes(CONF_VERSION_KEY)) == null) { - db.put(bytes(CONF_VERSION_KEY), - bytes(String.valueOf(System.currentTimeMillis()))); - } } catch (NativeDB.DBException e) { if (e.isNotFound() || e.getMessage().contains(" does not exist ")) { LOG.info("Creating conf database at " + dbfile); @@ -167,8 +158,6 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore { initBatch.put(bytes(kv.getKey()), bytes(kv.getValue())); } db.write(initBatch); - db.put(bytes(CONF_VERSION_KEY), - bytes(String.valueOf(System.currentTimeMillis()))); } catch (DBException dbErr) { throw new IOException(dbErr.getMessage(), dbErr); } @@ -226,8 +215,6 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore { } } db.write(updateBatch); - db.put(bytes(CONF_VERSION_KEY), - bytes(String.valueOf(System.currentTimeMillis()))); pendingMutation = null; } @@ -263,8 +250,7 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore { Map.Entry<byte[], byte[]> entry = itr.next(); String key = new String(entry.getKey(), StandardCharsets.UTF_8); String value = new String(entry.getValue(), StandardCharsets.UTF_8); - if (key.equals(LOG_KEY) || key.equals(VERSION_KEY) || - key.equals(CONF_VERSION_KEY)) { + if (key.equals(LOG_KEY) || key.equals(VERSION_KEY)) { break; } config.set(key, value); @@ -273,13 +259,6 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore { } @Override - public long getConfigVersion() throws Exception { - String version = new String(db.get(bytes(CONF_VERSION_KEY)), - StandardCharsets.UTF_8); - return Long.parseLong(version); - } - - @Override public List<LogMutation> getConfirmedConfHistory(long fromId) { return null; // unimplemented } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java index f464b2c..41b9b25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java @@ -135,11 +135,6 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, } @Override - public long getConfigVersion() throws Exception { - return confStore.getConfigVersion(); - } - - @Override public ConfigurationMutationACLPolicy getAclMutationPolicy() { return aclMutationPolicy; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java index 6af11a3..334c962 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java @@ -133,12 +133,6 @@ public abstract class YarnConfigurationStore { public abstract void format() throws Exception; /** - * Get the last updated config version. - * @return Last updated config version. - */ - public abstract long getConfigVersion() throws Exception; - - /** * Get a list of confirmed configuration mutations starting from a given id. * @param fromId id from which to start getting mutations, inclusive * @return list of configuration mutations diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java index 5066d05..d3fab39 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java @@ -62,13 +62,11 @@ public class ZKConfigurationStore extends YarnConfigurationStore { private static final String LOGS_PATH = "LOGS"; private static final String CONF_STORE_PATH = "CONF_STORE"; private static final String FENCING_PATH = "FENCING"; - private static final String CONF_VERSION_PATH = "CONF_VERSION"; private String zkVersionPath; private String logsPath; private String confStorePath; private String fencingNodePath; - private String confVersionPath; @VisibleForTesting protected ZKCuratorManager zkManager; @@ -91,7 +89,6 @@ public class ZKConfigurationStore extends YarnConfigurationStore { this.logsPath = getNodePath(znodeParentPath, LOGS_PATH); this.confStorePath = getNodePath(znodeParentPath, CONF_STORE_PATH); this.fencingNodePath = getNodePath(znodeParentPath, FENCING_PATH); - this.confVersionPath = getNodePath(znodeParentPath, CONF_VERSION_PATH); zkManager.createRootDirRecursively(znodeParentPath, zkAcl); zkManager.delete(fencingNodePath); @@ -102,12 +99,6 @@ public class ZKConfigurationStore extends YarnConfigurationStore { serializeObject(new LinkedList<LogMutation>()), -1); } - if (!zkManager.exists(confVersionPath)) { - zkManager.create(confVersionPath); - zkManager.setData(confVersionPath, - String.valueOf(System.currentTimeMillis()), -1); - } - if (!zkManager.exists(confStorePath)) { zkManager.create(confStorePath); HashMap<String, String> mapSchedConf = new HashMap<>(); @@ -115,8 +106,6 @@ public class ZKConfigurationStore extends YarnConfigurationStore { mapSchedConf.put(entry.getKey(), entry.getValue()); } zkManager.setData(confStorePath, serializeObject(mapSchedConf), -1); - zkManager.setData(confVersionPath, - String.valueOf(System.currentTimeMillis()), -1); } } @@ -196,9 +185,6 @@ public class ZKConfigurationStore extends YarnConfigurationStore { } zkManager.safeSetData(confStorePath, serializeObject(mapConf), -1, zkAcl, fencingNodePath); - zkManager.setData(confVersionPath, - String.valueOf(System.currentTimeMillis()), -1); - } pendingMutation = null; } @@ -228,11 +214,6 @@ public class ZKConfigurationStore extends YarnConfigurationStore { } @Override - public long getConfigVersion() throws Exception { - return Long.parseLong(zkManager.getStringData(confVersionPath)); - } - - @Override public List<LogMutation> getConfirmedConfHistory(long fromId) { return null; // unimplemented } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java index ab48140..6cc1e29 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java @@ -51,9 +51,6 @@ public final class RMWSConsts { /** Path for {@code RMWebServices#formatSchedulerConfiguration}. */ public static final String FORMAT_SCHEDULER_CONF = "/scheduler-conf/format"; - /** Path for {@code RMWebServices#getSchedulerConfigurationVersion}. */ - public static final String SCHEDULER_CONF_VERSION = "/scheduler-conf/version"; - /** Path for {@code RMWebServiceProtocol#dumpSchedulerLogs}. */ public static final String SCHEDULER_LOGS = "/scheduler/logs"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 7e8e06a..d1e04fa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -2590,7 +2590,7 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol { } } else { return Response.status(Status.BAD_REQUEST) - .entity("Scheduler Configuration format only supported by " + + .entity("Configuration change only supported by " + "MutableConfScheduler.").build(); } } @@ -2681,38 +2681,6 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol { } @GET - @Path(RMWSConsts.SCHEDULER_CONF_VERSION) - @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, - MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) - public Response getSchedulerConfigurationVersion(@Context - HttpServletRequest hsr) throws AuthorizationException { - // Only admin user is allowed to get scheduler conf version - UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); - initForWritableEndpoints(callerUGI, true); - - ResourceScheduler scheduler = rm.getResourceScheduler(); - if (scheduler instanceof MutableConfScheduler - && ((MutableConfScheduler) scheduler).isConfigurationMutable()) { - MutableConfigurationProvider mutableConfigurationProvider = - ((MutableConfScheduler) scheduler).getMutableConfProvider(); - - try { - long configVersion = mutableConfigurationProvider - .getConfigVersion(); - return Response.status(Status.OK).entity(configVersion).build(); - } catch (Exception e) { - LOG.error("Exception thrown when fetching configuration version.", e); - return Response.status(Status.BAD_REQUEST).entity(e.getMessage()) - .build(); - } - } else { - return Response.status(Status.BAD_REQUEST) - .entity("Configuration Version only supported by " - + "MutableConfScheduler.").build(); - } - } - - @GET @Path(RMWSConsts.CHECK_USER_ACCESS_TO_QUEUE) @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java index d6edd72..f71c4e7a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java @@ -138,21 +138,6 @@ public class TestZKConfigurationStore extends ConfigurationStoreBaseTest { } @Test - public void testGetConfigurationVersion() throws Exception { - confStore.initialize(conf, schedConf, rmContext); - long v1 = confStore.getConfigVersion(); - Thread.sleep(2000); - Map<String, String> update = new HashMap<>(); - update.put("keyver", "valver"); - YarnConfigurationStore.LogMutation mutation = - new YarnConfigurationStore.LogMutation(update, TEST_USER); - confStore.logMutation(mutation); - confStore.confirmMutation(true); - long v2 = confStore.getConfigVersion(); - assertTrue(v2 > v1); - } - - @Test public void testPersistUpdatedConfiguration() throws Exception { confStore.initialize(conf, schedConf, rmContext); assertNull(confStore.retrieve().get("key")); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org