hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sun...@apache.org
Subject [hadoop] branch trunk updated: YARN-9873. Mutation API Config Change need to update Version Number. Contributed by Prabhu Joseph
Date Wed, 09 Oct 2019 10:23:28 GMT
This is an automated email from the ASF dual-hosted git repository.

sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
     new be901f4  YARN-9873. Mutation API Config Change need to update Version Number. Contributed
by Prabhu Joseph
be901f4 is described below

commit be901f49628a553cfe6a3942478cb528cce2b266
Author: Sunil G <sunilg@apache.org>
AuthorDate: Wed Oct 9 15:53:14 2019 +0530

    YARN-9873. Mutation API Config Change need to update Version Number. Contributed by Prabhu
Joseph
---
 .../scheduler/MutableConfigurationProvider.java    |  6 +++
 .../conf/FSSchedulerConfigurationStore.java        | 43 ++++++++++++++++-
 .../capacity/conf/InMemoryConfigurationStore.java  |  8 ++++
 .../capacity/conf/LeveldbConfigurationStore.java   | 54 +++++++++++++++++++---
 .../conf/MutableCSConfigurationProvider.java       |  5 ++
 .../capacity/conf/YarnConfigurationStore.java      |  6 +++
 .../capacity/conf/ZKConfigurationStore.java        | 18 ++++++++
 .../server/resourcemanager/webapp/RMWSConsts.java  |  3 ++
 .../resourcemanager/webapp/RMWebServices.java      | 36 ++++++++++++++-
 .../webapp/dao/ConfigVersionInfo.java              | 44 ++++++++++++++++++
 .../conf/TestFSSchedulerConfigurationStore.java    | 12 +++--
 .../capacity/conf/TestZKConfigurationStore.java    | 15 ++++++
 .../TestRMWebServicesConfigurationMutation.java    | 19 ++++++++
 13 files changed, 258 insertions(+), 11 deletions(-)

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index 9e843df..eff8aa8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -65,6 +65,12 @@ public interface MutableConfigurationProvider {
    */
   Configuration getConfiguration();
 
+  /**
+   * Get the last updated scheduler config version.
+   * @return Last updated scheduler config version.
+   */
+  long getConfigVersion() throws Exception;
+
   void formatConfigurationInStore(Configuration conf) throws Exception;
 
   /**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
index 80053be..464ef14 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
@@ -29,6 +29,7 @@ import com.google.gson.GsonBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -62,6 +63,7 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore
{
   private volatile Configuration schedConf;
   private volatile Configuration oldConf;
   private Path tempConfigPath;
+  private Path configVersionFile;
 
   @Override
   public void initialize(Configuration conf, Configuration vSchedConf,
@@ -99,9 +101,17 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore
{
       }
     }
 
+    this.configVersionFile = new Path(schedulerConfPathStr, "ConfigVersion");
+    if (!fileSystem.exists(configVersionFile)) {
+      fileSystem.createNewFile(configVersionFile);
+      writeConfigVersion(0L);
+    }
+
     // create capacity-schedule.xml.ts file if not existing
     if (this.getConfigFileInputStream() == null) {
       writeConfigurationToFileSystem(vSchedConf);
+      long configVersion = getConfigVersion() + 1L;
+      writeConfigVersion(configVersion);
     }
 
     this.schedConf = this.getConfigurationFromFileSystem();
@@ -141,6 +151,8 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore
{
     }
     if (isValid) {
       finalizeFileSystemFile();
+      long configVersion = getConfigVersion() + 1L;
+      writeConfigVersion(configVersion);
     } else {
       schedConf = oldConf;
       removeTmpConfigFile();
@@ -158,7 +170,15 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore
{
 
   @Override
   public void format() throws Exception {
-    fileSystem.delete(schedulerConfDir, true);
+    FileStatus[] fileStatuses = fileSystem.listStatus(this.schedulerConfDir,
+        this.configFilePathFilter);
+    if (fileStatuses == null) {
+      return;
+    }
+    for (int i = 0; i < fileStatuses.length; i++) {
+      fileSystem.delete(fileStatuses[i].getPath(), false);
+      LOG.info("delete config file " + fileStatuses[i].getPath());
+    }
   }
 
   private Path getFinalConfigPath(Path tempPath) {
@@ -222,6 +242,27 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore
{
     return fileStatuses[fileStatuses.length - 1].getPath();
   }
 
+  private void writeConfigVersion(long configVersion) throws IOException {
+    try (FSDataOutputStream out = fileSystem.create(configVersionFile, true)) {
+      out.writeLong(configVersion);
+    } catch (IOException e) {
+      LOG.info("Failed to write config version at {}", configVersionFile, e);
+      throw e;
+    }
+  }
+
+  @Override
+  public long getConfigVersion() throws Exception {
+    try (FSDataInputStream in = fileSystem.open(configVersionFile)) {
+      return in.readLong();
+    } catch (IOException e) {
+      LOG.info("Failed to read config version at {}", configVersionFile, e);
+      throw e;
+    }
+  }
+
+
+
   @VisibleForTesting
   private Path writeTmpConfig(Configuration vSchedConf) throws IOException {
     long start = Time.monotonicNow();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
index 4871443..47dd6bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
@@ -33,11 +33,13 @@ public class InMemoryConfigurationStore extends YarnConfigurationStore
{
 
   private Configuration schedConf;
   private LogMutation pendingMutation;
+  private long configVersion;
 
   @Override
   public void initialize(Configuration conf, Configuration schedConf,
       RMContext rmContext) {
     this.schedConf = schedConf;
+    this.configVersion = 1L;
   }
 
   @Override
@@ -56,6 +58,7 @@ public class InMemoryConfigurationStore extends YarnConfigurationStore {
           schedConf.set(kv.getKey(), kv.getValue());
         }
       }
+      this.configVersion = this.configVersion + 1L;
     }
     pendingMutation = null;
   }
@@ -71,6 +74,11 @@ public class InMemoryConfigurationStore extends YarnConfigurationStore
{
   }
 
   @Override
+  public long getConfigVersion() {
+    return configVersion;
+  }
+
+  @Override
   public List<LogMutation> getConfirmedConfHistory(long fromId) {
     // Unimplemented.
     return null;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
index 743d7ef..2966c94 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
@@ -68,8 +68,11 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
   private static final String DB_NAME = "yarn-conf-store";
   private static final String LOG_KEY = "log";
   private static final String VERSION_KEY = "version";
+  private static final String CONF_VERSION_NAME = "conf-version-store";
+  private static final String CONF_VERSION_KEY = "conf-version";
 
   private DB db;
+  private DB versiondb;
   private long maxLogs;
   private Configuration conf;
   private LogMutation pendingMutation;
@@ -102,11 +105,11 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore
{
   public void format() throws Exception {
     close();
     FileSystem fs = FileSystem.getLocal(conf);
-    fs.delete(getStorageDir(), true);
+    fs.delete(getStorageDir(DB_NAME), true);
   }
 
   private void initDatabase(Configuration config) throws Exception {
-    Path storeRoot = createStorageDir();
+    Path storeRoot = createStorageDir(DB_NAME);
     Options options = new Options();
     options.createIfMissing(false);
     options.comparator(new DBComparator() {
@@ -142,6 +145,29 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore
{
       }
     });
 
+    Path confVersion = createStorageDir(CONF_VERSION_NAME);
+    Options confOptions = new Options();
+    confOptions.createIfMissing(false);
+    LOG.info("Using conf version at " + confVersion);
+    File confVersionFile = new File(confVersion.toString());
+    try {
+      versiondb = JniDBFactory.factory.open(confVersionFile, confOptions);
+    } catch (NativeDB.DBException e) {
+      if (e.isNotFound() || e.getMessage().contains(" does not exist ")) {
+        LOG.info("Creating conf version at " + confVersionFile);
+        confOptions.createIfMissing(true);
+        try {
+          versiondb = JniDBFactory.factory.open(confVersionFile, confOptions);
+          versiondb.put(bytes(CONF_VERSION_KEY), bytes(String.valueOf(0)));
+        } catch (DBException dbErr) {
+          throw new IOException(dbErr.getMessage(), dbErr);
+        }
+      } else {
+        throw e;
+      }
+    }
+
+
     LOG.info("Using conf database at " + storeRoot);
     File dbfile = new File(storeRoot.toString());
     try {
@@ -158,6 +184,9 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore
{
             initBatch.put(bytes(kv.getKey()), bytes(kv.getValue()));
           }
           db.write(initBatch);
+          long configVersion = getConfigVersion() + 1L;
+          versiondb.put(bytes(CONF_VERSION_KEY),
+              bytes(String.valueOf(configVersion)));
         } catch (DBException dbErr) {
           throw new IOException(dbErr.getMessage(), dbErr);
         }
@@ -167,20 +196,20 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore
{
     }
   }
 
-  private Path createStorageDir() throws IOException {
-    Path root = getStorageDir();
+  private Path createStorageDir(String storageName) throws IOException {
+    Path root = getStorageDir(storageName);
     FileSystem fs = FileSystem.getLocal(conf);
     fs.mkdirs(root, new FsPermission((short) 0700));
     return root;
   }
 
-  private Path getStorageDir() throws IOException {
+  private Path getStorageDir(String storageName) throws IOException {
     String storePath = conf.get(YarnConfiguration.RM_SCHEDCONF_STORE_PATH);
     if (storePath == null) {
       throw new IOException("No store location directory configured in " +
           YarnConfiguration.RM_SCHEDCONF_STORE_PATH);
     }
-    return new Path(storePath, DB_NAME);
+    return new Path(storePath, storageName);
   }
 
   @Override
@@ -188,6 +217,9 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore
{
     if (db != null) {
       db.close();
     }
+    if (versiondb != null) {
+      versiondb.close();
+    }
   }
 
   @Override
@@ -213,6 +245,9 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore
{
           updateBatch.put(bytes(changes.getKey()), bytes(changes.getValue()));
         }
       }
+      long configVersion = getConfigVersion() + 1L;
+      versiondb.put(bytes(CONF_VERSION_KEY),
+          bytes(String.valueOf(configVersion)));
     }
     db.write(updateBatch);
     pendingMutation = null;
@@ -259,6 +294,13 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore
{
   }
 
   @Override
+  public long getConfigVersion() {
+    String version = new String(versiondb.get(bytes(CONF_VERSION_KEY)),
+        StandardCharsets.UTF_8);
+    return Long.parseLong(version);
+  }
+
+  @Override
   public List<LogMutation> getConfirmedConfHistory(long fromId) {
     return null; // unimplemented
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index 41b9b25..f464b2c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -135,6 +135,11 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
   }
 
   @Override
+  public long getConfigVersion() throws Exception {
+    return confStore.getConfigVersion();
+  }
+
+  @Override
   public ConfigurationMutationACLPolicy getAclMutationPolicy() {
     return aclMutationPolicy;
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
index 334c962..6af11a3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -133,6 +133,12 @@ public abstract class YarnConfigurationStore {
   public abstract void format() throws Exception;
 
   /**
+   * Get the last updated config version.
+   * @return Last updated config version.
+   */
+  public abstract long getConfigVersion() throws Exception;
+
+  /**
    * Get a list of confirmed configuration mutations starting from a given id.
    * @param fromId id from which to start getting mutations, inclusive
    * @return list of configuration mutations
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
index d3fab39..1aee415 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
@@ -62,11 +62,13 @@ public class ZKConfigurationStore extends YarnConfigurationStore {
   private static final String LOGS_PATH = "LOGS";
   private static final String CONF_STORE_PATH = "CONF_STORE";
   private static final String FENCING_PATH = "FENCING";
+  private static final String CONF_VERSION_PATH = "CONF_VERSION";
 
   private String zkVersionPath;
   private String logsPath;
   private String confStorePath;
   private String fencingNodePath;
+  private String confVersionPath;
 
   @VisibleForTesting
   protected ZKCuratorManager zkManager;
@@ -89,6 +91,7 @@ public class ZKConfigurationStore extends YarnConfigurationStore {
     this.logsPath = getNodePath(znodeParentPath, LOGS_PATH);
     this.confStorePath = getNodePath(znodeParentPath, CONF_STORE_PATH);
     this.fencingNodePath = getNodePath(znodeParentPath, FENCING_PATH);
+    this.confVersionPath = getNodePath(znodeParentPath, CONF_VERSION_PATH);
 
     zkManager.createRootDirRecursively(znodeParentPath, zkAcl);
     zkManager.delete(fencingNodePath);
@@ -99,6 +102,11 @@ public class ZKConfigurationStore extends YarnConfigurationStore {
           serializeObject(new LinkedList<LogMutation>()), -1);
     }
 
+    if (!zkManager.exists(confVersionPath)) {
+      zkManager.create(confVersionPath);
+      zkManager.setData(confVersionPath, String.valueOf(0), -1);
+    }
+
     if (!zkManager.exists(confStorePath)) {
       zkManager.create(confStorePath);
       HashMap<String, String> mapSchedConf = new HashMap<>();
@@ -106,6 +114,8 @@ public class ZKConfigurationStore extends YarnConfigurationStore {
         mapSchedConf.put(entry.getKey(), entry.getValue());
       }
       zkManager.setData(confStorePath, serializeObject(mapSchedConf), -1);
+      long configVersion = getConfigVersion() + 1L;
+      zkManager.setData(confVersionPath, String.valueOf(configVersion), -1);
     }
   }
 
@@ -185,6 +195,9 @@ public class ZKConfigurationStore extends YarnConfigurationStore {
       }
       zkManager.safeSetData(confStorePath, serializeObject(mapConf), -1,
           zkAcl, fencingNodePath);
+      long configVersion = getConfigVersion() + 1L;
+      zkManager.setData(confVersionPath, String.valueOf(configVersion), -1);
+
     }
     pendingMutation = null;
   }
@@ -214,6 +227,11 @@ public class ZKConfigurationStore extends YarnConfigurationStore {
   }
 
   @Override
+  public long getConfigVersion() throws Exception {
+    return Long.parseLong(zkManager.getStringData(confVersionPath));
+  }
+
+  @Override
   public List<LogMutation> getConfirmedConfHistory(long fromId) {
     return null; // unimplemented
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java
index 6cc1e29..ab48140 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java
@@ -51,6 +51,9 @@ public final class RMWSConsts {
   /** Path for {@code RMWebServices#formatSchedulerConfiguration}. */
   public static final String FORMAT_SCHEDULER_CONF = "/scheduler-conf/format";
 
+  /** Path for {@code RMWebServices#getSchedulerConfigurationVersion}. */
+  public static final String SCHEDULER_CONF_VERSION = "/scheduler-conf/version";
+
   /** Path for {@code RMWebServiceProtocol#dumpSchedulerLogs}. */
   public static final String SCHEDULER_LOGS = "/scheduler/logs";
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index d1e04fa..bb77dbd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -196,6 +196,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionIn
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfigVersionInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfInfo;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
@@ -2590,7 +2591,7 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol
{
       }
     } else {
       return Response.status(Status.BAD_REQUEST)
-          .entity("Configuration change only supported by " +
+          .entity("Scheduler Configuration format only supported by " +
           "MutableConfScheduler.").build();
     }
   }
@@ -2681,6 +2682,39 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol
{
   }
 
   @GET
+  @Path(RMWSConsts.SCHEDULER_CONF_VERSION)
+  @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
+       MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+  public Response getSchedulerConfigurationVersion(@Context
+      HttpServletRequest hsr) throws AuthorizationException {
+    // Only admin user is allowed to get scheduler conf version
+    UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
+    initForWritableEndpoints(callerUGI, true);
+
+    ResourceScheduler scheduler = rm.getResourceScheduler();
+    if (scheduler instanceof MutableConfScheduler
+        && ((MutableConfScheduler) scheduler).isConfigurationMutable()) {
+      MutableConfigurationProvider mutableConfigurationProvider =
+          ((MutableConfScheduler) scheduler).getMutableConfProvider();
+
+      try {
+        long configVersion = mutableConfigurationProvider
+            .getConfigVersion();
+        return Response.status(Status.OK)
+            .entity(new ConfigVersionInfo(configVersion)).build();
+      } catch (Exception e) {
+        LOG.error("Exception thrown when fetching configuration version.", e);
+        return Response.status(Status.BAD_REQUEST).entity(e.getMessage())
+            .build();
+      }
+    } else {
+      return Response.status(Status.BAD_REQUEST)
+          .entity("Configuration Version only supported by "
+          + "MutableConfScheduler.").build();
+    }
+  }
+
+  @GET
   @Path(RMWSConsts.CHECK_USER_ACCESS_TO_QUEUE)
   @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
                 MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfigVersionInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfigVersionInfo.java
new file mode 100644
index 0000000..50a2728
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfigVersionInfo.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * Version of Scheduler Config.
+ */
+@XmlRootElement(name = "configversion")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class ConfigVersionInfo {
+
+  private long versionID;
+
+  public ConfigVersionInfo() {
+  } // JAXB needs this
+
+  public ConfigVersionInfo(long version) {
+    this.versionID = version;
+  }
+
+  public long getVersionID() {
+    return this.versionID;
+  }
+
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java
index f3d5e74..33596c3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java
@@ -37,7 +37,6 @@ import org.junit.Test;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertFalse;
 
 
 /**
@@ -140,7 +139,6 @@ public class TestFSSchedulerConfigurationStore {
 
   @Test
   public void testFormatConfiguration() throws Exception {
-    assertTrue(testSchedulerConfigurationDir.exists());
     Configuration schedulerConf = new Configuration();
     schedulerConf.set("a", "a");
     writeConf(schedulerConf);
@@ -148,7 +146,15 @@ public class TestFSSchedulerConfigurationStore {
     Configuration storedConfig = configurationStore.retrieve();
     assertEquals("a", storedConfig.get("a"));
     configurationStore.format();
-    assertFalse(testSchedulerConfigurationDir.exists());
+    boolean exceptionCaught = false;
+    try {
+      storedConfig = configurationStore.retrieve();
+    } catch (IOException e) {
+      if (e.getMessage().contains("no capacity scheduler file in")) {
+        exceptionCaught = true;
+      }
+    }
+    assertTrue(exceptionCaught);
   }
 
   @Test
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
index f71c4e7a..eae80d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
@@ -138,6 +138,21 @@ public class TestZKConfigurationStore extends ConfigurationStoreBaseTest
{
   }
 
   @Test
+  public void testGetConfigurationVersion() throws Exception {
+    confStore.initialize(conf, schedConf, rmContext);
+    long v1 = confStore.getConfigVersion();
+    assertEquals(1, v1);
+    Map<String, String> update = new HashMap<>();
+    update.put("keyver", "valver");
+    YarnConfigurationStore.LogMutation mutation =
+        new YarnConfigurationStore.LogMutation(update, TEST_USER);
+    confStore.logMutation(mutation);
+    confStore.confirmMutation(true);
+    long v2 = confStore.getConfigVersion();
+    assertEquals(2, v2);
+  }
+
+  @Test
   public void testPersistUpdatedConfiguration() throws Exception {
     confStore.initialize(conf, schedConf, rmContext);
     assertNull(confStore.retrieve().get("key"));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
index 67f83c8..c717d8b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
@@ -202,6 +202,25 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase
{
     assertEquals(3, orgConf.getQueues("root").length);
   }
 
+  private long getConfigVersion() throws Exception {
+    WebResource r = resource();
+    ClientResponse response = r.path("ws").path("v1").path("cluster")
+        .queryParam("user.name", userName)
+        .path(RMWSConsts.SCHEDULER_CONF_VERSION)
+        .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(Status.OK.getStatusCode(), response.getStatus());
+
+    JSONObject json = response.getEntity(JSONObject.class);
+    return Long.parseLong(json.get("versionID").toString());
+  }
+
+  @Test
+  public void testSchedulerConfigVersion() throws Exception {
+    assertEquals(1, getConfigVersion());
+    testAddNestedQueue();
+    assertEquals(2, getConfigVersion());
+  }
+
   @Test
   public void testAddNestedQueue() throws Exception {
     CapacitySchedulerConfiguration orgConf = getSchedulerConf();


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message