This is an automated email from the ASF dual-hosted git repository. sunilg pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push: new be901f4 YARN-9873. Mutation API Config Change need to update Version Number. Contributed by Prabhu Joseph be901f4 is described below commit be901f49628a553cfe6a3942478cb528cce2b266 Author: Sunil G <sun...@apache.org> AuthorDate: Wed Oct 9 15:53:14 2019 +0530 YARN-9873. Mutation API Config Change need to update Version Number. Contributed by Prabhu Joseph --- .../scheduler/MutableConfigurationProvider.java | 6 +++ .../conf/FSSchedulerConfigurationStore.java | 43 ++++++++++++++++- .../capacity/conf/InMemoryConfigurationStore.java | 8 ++++ .../capacity/conf/LeveldbConfigurationStore.java | 54 +++++++++++++++++++--- .../conf/MutableCSConfigurationProvider.java | 5 ++ .../capacity/conf/YarnConfigurationStore.java | 6 +++ .../capacity/conf/ZKConfigurationStore.java | 18 ++++++++ .../server/resourcemanager/webapp/RMWSConsts.java | 3 ++ .../resourcemanager/webapp/RMWebServices.java | 36 ++++++++++++++- .../webapp/dao/ConfigVersionInfo.java | 44 ++++++++++++++++++ .../conf/TestFSSchedulerConfigurationStore.java | 12 +++-- .../capacity/conf/TestZKConfigurationStore.java | 15 ++++++ .../TestRMWebServicesConfigurationMutation.java | 19 ++++++++ 13 files changed, 258 insertions(+), 11 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java index 9e843df..eff8aa8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java @@ -65,6 +65,12 @@ public interface MutableConfigurationProvider { */ Configuration getConfiguration(); + /** + * Get the last updated scheduler config version. + * @return Last updated scheduler config version. + */ + long getConfigVersion() throws Exception; + void formatConfigurationInStore(Configuration conf) throws Exception; /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java index 80053be..464ef14 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java @@ -29,6 +29,7 @@ import com.google.gson.GsonBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -62,6 +63,7 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore { private volatile Configuration schedConf; private volatile Configuration oldConf; private Path tempConfigPath; + private Path configVersionFile; @Override public void initialize(Configuration conf, Configuration vSchedConf, @@ -99,9 +101,17 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore { } } + this.configVersionFile = new Path(schedulerConfPathStr, "ConfigVersion"); + if (!fileSystem.exists(configVersionFile)) { + fileSystem.createNewFile(configVersionFile); + writeConfigVersion(0L); + } + // create capacity-schedule.xml.ts file if not existing if (this.getConfigFileInputStream() == null) { writeConfigurationToFileSystem(vSchedConf); + long configVersion = getConfigVersion() + 1L; + writeConfigVersion(configVersion); } this.schedConf = this.getConfigurationFromFileSystem(); @@ -141,6 +151,8 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore { } if (isValid) { finalizeFileSystemFile(); + long configVersion = getConfigVersion() + 1L; + writeConfigVersion(configVersion); } else { schedConf = oldConf; removeTmpConfigFile(); @@ -158,7 +170,15 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore { @Override public void format() throws Exception { - fileSystem.delete(schedulerConfDir, true); + FileStatus[] fileStatuses = fileSystem.listStatus(this.schedulerConfDir, + this.configFilePathFilter); + if (fileStatuses == null) { + return; + } + for (int i = 0; i < fileStatuses.length; i++) { + fileSystem.delete(fileStatuses[i].getPath(), false); + LOG.info("delete config file " + fileStatuses[i].getPath()); + } } private Path getFinalConfigPath(Path tempPath) { @@ -222,6 +242,27 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore { return fileStatuses[fileStatuses.length - 1].getPath(); } + private void writeConfigVersion(long configVersion) throws IOException { + try (FSDataOutputStream out = fileSystem.create(configVersionFile, true)) { + out.writeLong(configVersion); + } catch (IOException e) { + LOG.info("Failed to write config version at {}", configVersionFile, e); + throw e; + } + } + + @Override + public long getConfigVersion() throws Exception { + try (FSDataInputStream in = fileSystem.open(configVersionFile)) { + return in.readLong(); + } catch (IOException e) { + LOG.info("Failed to read config version at {}", configVersionFile, e); + throw e; + } + } + + + @VisibleForTesting private Path writeTmpConfig(Configuration vSchedConf) throws IOException { long start = Time.monotonicNow(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java index 4871443..47dd6bd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java @@ -33,11 +33,13 @@ public class InMemoryConfigurationStore extends YarnConfigurationStore { private Configuration schedConf; private LogMutation pendingMutation; + private long configVersion; @Override public void initialize(Configuration conf, Configuration schedConf, RMContext rmContext) { this.schedConf = schedConf; + this.configVersion = 1L; } @Override @@ -56,6 +58,7 @@ public class InMemoryConfigurationStore extends YarnConfigurationStore { schedConf.set(kv.getKey(), kv.getValue()); } } + this.configVersion = this.configVersion + 1L; } pendingMutation = null; } @@ -71,6 +74,11 @@ public class InMemoryConfigurationStore extends YarnConfigurationStore { } @Override + public long getConfigVersion() { + return configVersion; + } + + @Override public List<LogMutation> getConfirmedConfHistory(long fromId) { // Unimplemented. return null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java index 743d7ef..2966c94 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java @@ -68,8 +68,11 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore { private static final String DB_NAME = "yarn-conf-store"; private static final String LOG_KEY = "log"; private static final String VERSION_KEY = "version"; + private static final String CONF_VERSION_NAME = "conf-version-store"; + private static final String CONF_VERSION_KEY = "conf-version"; private DB db; + private DB versiondb; private long maxLogs; private Configuration conf; private LogMutation pendingMutation; @@ -102,11 +105,11 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore { public void format() throws Exception { close(); FileSystem fs = FileSystem.getLocal(conf); - fs.delete(getStorageDir(), true); + fs.delete(getStorageDir(DB_NAME), true); } private void initDatabase(Configuration config) throws Exception { - Path storeRoot = createStorageDir(); + Path storeRoot = createStorageDir(DB_NAME); Options options = new Options(); options.createIfMissing(false); options.comparator(new DBComparator() { @@ -142,6 +145,29 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore { } }); + Path confVersion = createStorageDir(CONF_VERSION_NAME); + Options confOptions = new Options(); + confOptions.createIfMissing(false); + LOG.info("Using conf version at " + confVersion); + File confVersionFile = new File(confVersion.toString()); + try { + versiondb = JniDBFactory.factory.open(confVersionFile, confOptions); + } catch (NativeDB.DBException e) { + if (e.isNotFound() || e.getMessage().contains(" does not exist ")) { + LOG.info("Creating conf version at " + confVersionFile); + confOptions.createIfMissing(true); + try { + versiondb = JniDBFactory.factory.open(confVersionFile, confOptions); + versiondb.put(bytes(CONF_VERSION_KEY), bytes(String.valueOf(0))); + } catch (DBException dbErr) { + throw new IOException(dbErr.getMessage(), dbErr); + } + } else { + throw e; + } + } + + LOG.info("Using conf database at " + storeRoot); File dbfile = new File(storeRoot.toString()); try { @@ -158,6 +184,9 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore { initBatch.put(bytes(kv.getKey()), bytes(kv.getValue())); } db.write(initBatch); + long configVersion = getConfigVersion() + 1L; + versiondb.put(bytes(CONF_VERSION_KEY), + bytes(String.valueOf(configVersion))); } catch (DBException dbErr) { throw new IOException(dbErr.getMessage(), dbErr); } @@ -167,20 +196,20 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore { } } - private Path createStorageDir() throws IOException { - Path root = getStorageDir(); + private Path createStorageDir(String storageName) throws IOException { + Path root = getStorageDir(storageName); FileSystem fs = FileSystem.getLocal(conf); fs.mkdirs(root, new FsPermission((short) 0700)); return root; } - private Path getStorageDir() throws IOException { + private Path getStorageDir(String storageName) throws IOException { String storePath = conf.get(YarnConfiguration.RM_SCHEDCONF_STORE_PATH); if (storePath == null) { throw new IOException("No store location directory configured in " + YarnConfiguration.RM_SCHEDCONF_STORE_PATH); } - return new Path(storePath, DB_NAME); + return new Path(storePath, storageName); } @Override @@ -188,6 +217,9 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore { if (db != null) { db.close(); } + if (versiondb != null) { + versiondb.close(); + } } @Override @@ -213,6 +245,9 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore { updateBatch.put(bytes(changes.getKey()), bytes(changes.getValue())); } } + long configVersion = getConfigVersion() + 1L; + versiondb.put(bytes(CONF_VERSION_KEY), + bytes(String.valueOf(configVersion))); } db.write(updateBatch); pendingMutation = null; @@ -259,6 +294,13 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore { } @Override + public long getConfigVersion() { + String version = new String(versiondb.get(bytes(CONF_VERSION_KEY)), + StandardCharsets.UTF_8); + return Long.parseLong(version); + } + + @Override public List<LogMutation> getConfirmedConfHistory(long fromId) { return null; // unimplemented } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java index 41b9b25..f464b2c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java @@ -135,6 +135,11 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, } @Override + public long getConfigVersion() throws Exception { + return confStore.getConfigVersion(); + } + + @Override public ConfigurationMutationACLPolicy getAclMutationPolicy() { return aclMutationPolicy; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java index 334c962..6af11a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java @@ -133,6 +133,12 @@ public abstract class YarnConfigurationStore { public abstract void format() throws Exception; /** + * Get the last updated config version. + * @return Last updated config version. + */ + public abstract long getConfigVersion() throws Exception; + + /** * Get a list of confirmed configuration mutations starting from a given id. * @param fromId id from which to start getting mutations, inclusive * @return list of configuration mutations diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java index d3fab39..1aee415 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java @@ -62,11 +62,13 @@ public class ZKConfigurationStore extends YarnConfigurationStore { private static final String LOGS_PATH = "LOGS"; private static final String CONF_STORE_PATH = "CONF_STORE"; private static final String FENCING_PATH = "FENCING"; + private static final String CONF_VERSION_PATH = "CONF_VERSION"; private String zkVersionPath; private String logsPath; private String confStorePath; private String fencingNodePath; + private String confVersionPath; @VisibleForTesting protected ZKCuratorManager zkManager; @@ -89,6 +91,7 @@ public class ZKConfigurationStore extends YarnConfigurationStore { this.logsPath = getNodePath(znodeParentPath, LOGS_PATH); this.confStorePath = getNodePath(znodeParentPath, CONF_STORE_PATH); this.fencingNodePath = getNodePath(znodeParentPath, FENCING_PATH); + this.confVersionPath = getNodePath(znodeParentPath, CONF_VERSION_PATH); zkManager.createRootDirRecursively(znodeParentPath, zkAcl); zkManager.delete(fencingNodePath); @@ -99,6 +102,11 @@ public class ZKConfigurationStore extends YarnConfigurationStore { serializeObject(new LinkedList<LogMutation>()), -1); } + if (!zkManager.exists(confVersionPath)) { + zkManager.create(confVersionPath); + zkManager.setData(confVersionPath, String.valueOf(0), -1); + } + if (!zkManager.exists(confStorePath)) { zkManager.create(confStorePath); HashMap<String, String> mapSchedConf = new HashMap<>(); @@ -106,6 +114,8 @@ public class ZKConfigurationStore extends YarnConfigurationStore { mapSchedConf.put(entry.getKey(), entry.getValue()); } zkManager.setData(confStorePath, serializeObject(mapSchedConf), -1); + long configVersion = getConfigVersion() + 1L; + zkManager.setData(confVersionPath, String.valueOf(configVersion), -1); } } @@ -185,6 +195,9 @@ public class ZKConfigurationStore extends YarnConfigurationStore { } zkManager.safeSetData(confStorePath, serializeObject(mapConf), -1, zkAcl, fencingNodePath); + long configVersion = getConfigVersion() + 1L; + zkManager.setData(confVersionPath, String.valueOf(configVersion), -1); + } pendingMutation = null; } @@ -214,6 +227,11 @@ public class ZKConfigurationStore extends YarnConfigurationStore { } @Override + public long getConfigVersion() throws Exception { + return Long.parseLong(zkManager.getStringData(confVersionPath)); + } + + @Override public List<LogMutation> getConfirmedConfHistory(long fromId) { return null; // unimplemented } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java index 6cc1e29..ab48140 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java @@ -51,6 +51,9 @@ public final class RMWSConsts { /** Path for {@code RMWebServices#formatSchedulerConfiguration}. */ public static final String FORMAT_SCHEDULER_CONF = "/scheduler-conf/format"; + /** Path for {@code RMWebServices#getSchedulerConfigurationVersion}. */ + public static final String SCHEDULER_CONF_VERSION = "/scheduler-conf/version"; + /** Path for {@code RMWebServiceProtocol#dumpSchedulerLogs}. */ public static final String SCHEDULER_LOGS = "/scheduler/logs"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index d1e04fa..bb77dbd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -196,6 +196,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionIn import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfigVersionInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfInfo; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -2590,7 +2591,7 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol { } } else { return Response.status(Status.BAD_REQUEST) - .entity("Configuration change only supported by " + + .entity("Scheduler Configuration format only supported by " + "MutableConfScheduler.").build(); } } @@ -2681,6 +2682,39 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol { } @GET + @Path(RMWSConsts.SCHEDULER_CONF_VERSION) + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + public Response getSchedulerConfigurationVersion(@Context + HttpServletRequest hsr) throws AuthorizationException { + // Only admin user is allowed to get scheduler conf version + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + initForWritableEndpoints(callerUGI, true); + + ResourceScheduler scheduler = rm.getResourceScheduler(); + if (scheduler instanceof MutableConfScheduler + && ((MutableConfScheduler) scheduler).isConfigurationMutable()) { + MutableConfigurationProvider mutableConfigurationProvider = + ((MutableConfScheduler) scheduler).getMutableConfProvider(); + + try { + long configVersion = mutableConfigurationProvider + .getConfigVersion(); + return Response.status(Status.OK) + .entity(new ConfigVersionInfo(configVersion)).build(); + } catch (Exception e) { + LOG.error("Exception thrown when fetching configuration version.", e); + return Response.status(Status.BAD_REQUEST).entity(e.getMessage()) + .build(); + } + } else { + return Response.status(Status.BAD_REQUEST) + .entity("Configuration Version only supported by " + + "MutableConfScheduler.").build(); + } + } + + @GET @Path(RMWSConsts.CHECK_USER_ACCESS_TO_QUEUE) @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfigVersionInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfigVersionInfo.java new file mode 100644 index 0000000..50a2728 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfigVersionInfo.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * Version of Scheduler Config. + */ +@XmlRootElement(name = "configversion") +@XmlAccessorType(XmlAccessType.FIELD) +public class ConfigVersionInfo { + + private long versionID; + + public ConfigVersionInfo() { + } // JAXB needs this + + public ConfigVersionInfo(long version) { + this.versionID = version; + } + + public long getVersionID() { + return this.versionID; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java index f3d5e74..33596c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java @@ -37,7 +37,6 @@ import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertFalse; /** @@ -140,7 +139,6 @@ public class TestFSSchedulerConfigurationStore { @Test public void testFormatConfiguration() throws Exception { - assertTrue(testSchedulerConfigurationDir.exists()); Configuration schedulerConf = new Configuration(); schedulerConf.set("a", "a"); writeConf(schedulerConf); @@ -148,7 +146,15 @@ public class TestFSSchedulerConfigurationStore { Configuration storedConfig = configurationStore.retrieve(); assertEquals("a", storedConfig.get("a")); configurationStore.format(); - assertFalse(testSchedulerConfigurationDir.exists()); + boolean exceptionCaught = false; + try { + storedConfig = configurationStore.retrieve(); + } catch (IOException e) { + if (e.getMessage().contains("no capacity scheduler file in")) { + exceptionCaught = true; + } + } + assertTrue(exceptionCaught); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java index f71c4e7a..eae80d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java @@ -138,6 +138,21 @@ public class TestZKConfigurationStore extends ConfigurationStoreBaseTest { } @Test + public void testGetConfigurationVersion() throws Exception { + confStore.initialize(conf, schedConf, rmContext); + long v1 = confStore.getConfigVersion(); + assertEquals(1, v1); + Map<String, String> update = new HashMap<>(); + update.put("keyver", "valver"); + YarnConfigurationStore.LogMutation mutation = + new YarnConfigurationStore.LogMutation(update, TEST_USER); + confStore.logMutation(mutation); + confStore.confirmMutation(true); + long v2 = confStore.getConfigVersion(); + assertEquals(2, v2); + } + + @Test public void testPersistUpdatedConfiguration() throws Exception { confStore.initialize(conf, schedConf, rmContext); assertNull(confStore.retrieve().get("key")); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java index 67f83c8..c717d8b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java @@ -202,6 +202,25 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { assertEquals(3, orgConf.getQueues("root").length); } + private long getConfigVersion() throws Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .queryParam("user.name", userName) + .path(RMWSConsts.SCHEDULER_CONF_VERSION) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + + JSONObject json = response.getEntity(JSONObject.class); + return Long.parseLong(json.get("versionID").toString()); + } + + @Test + public void testSchedulerConfigVersion() throws Exception { + assertEquals(1, getConfigVersion()); + testAddNestedQueue(); + assertEquals(2, getConfigVersion()); + } + @Test public void testAddNestedQueue() throws Exception { CapacitySchedulerConfiguration orgConf = getSchedulerConf(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org