hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sa...@apache.org
Subject svn commit: r1528203 - in /hadoop/common/branches/branch-2/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ hadoop-yarn/hadoop-yarn-ser...
Date Tue, 01 Oct 2013 20:16:30 GMT
Author: sandy
Date: Tue Oct  1 20:16:30 2013
New Revision: 1528203

URL: http://svn.apache.org/r1528203
Log:
YARN-1228. Clean up Fair Scheduler configuration loading. (Sandy Ryza)

Added:
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/test-fair-scheduler.xml
      - copied unchanged from r1528201, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/test-fair-scheduler.xml
Modified:
    hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1528203&r1=1528202&r2=1528203&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Tue Oct  1 20:16:30 2013
@@ -70,6 +70,8 @@ Release 2.1.2 - UNRELEASED
     YARN-899. Added back queue level administrator-acls so that there is no
     regression w.r.t 1.x. (Xuan Gong via vinodkv)
 
+    YARN-1228. Clean up Fair Scheduler configuration loading. (Sandy Ryza)
+
   OPTIMIZATIONS
 
   BUG FIXES

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java?rev=1528203&r1=1528202&r2=1528203&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
Tue Oct  1 20:16:30 2013
@@ -18,9 +18,12 @@
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
 
 import java.io.File;
+import java.net.URL;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.conf.Configuration;
@@ -33,6 +36,9 @@ import org.apache.hadoop.yarn.util.resou
 @Evolving
 public class FairSchedulerConfiguration extends Configuration {
 
+  public static final Log LOG = LogFactory.getLog(
+      FairSchedulerConfiguration.class.getName());
+  
   /** Increment request grant-able by the RM scheduler. 
    * These properties are looked up in the yarn-site.xml  */
   public static final String RM_SCHEDULER_INCREMENT_ALLOCATION_MB =
@@ -42,11 +48,10 @@ public class FairSchedulerConfiguration 
     YarnConfiguration.YARN_PREFIX + "scheduler.increment-allocation-vcores";
   public static final int DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES = 1;
   
-  public static final String FS_CONFIGURATION_FILE = "fair-scheduler.xml";
-
   private static final String CONF_PREFIX =  "yarn.scheduler.fair.";
 
   protected static final String ALLOCATION_FILE = CONF_PREFIX + "allocation.file";
+  protected static final String DEFAULT_ALLOCATION_FILE = "fair-scheduler.xml";
   protected static final String EVENT_LOG_DIR = "eventlog.dir";
 
   /** Whether to use the user name as the queue name (instead of "default") if
@@ -105,7 +110,6 @@ public class FairSchedulerConfiguration 
 
   public FairSchedulerConfiguration(Configuration conf) {
     super(conf);
-    addResource(FS_CONFIGURATION_FILE);
   }
 
   public Resource getMinimumAllocation() {
@@ -182,8 +186,28 @@ public class FairSchedulerConfiguration 
     return getBoolean(SIZE_BASED_WEIGHT, DEFAULT_SIZE_BASED_WEIGHT);
   }
 
-  public String getAllocationFile() {
-    return get(ALLOCATION_FILE);
+  /**
+   * Path to XML file containing allocations. If the
+   * path is relative, it is searched for in the
+   * classpath, but loaded like a regular File.
+   */
+  public File getAllocationFile() {
+    String allocFilePath = get(ALLOCATION_FILE, DEFAULT_ALLOCATION_FILE);
+    File allocFile = new File(allocFilePath);
+    if (!allocFile.isAbsolute()) {
+      URL url = Thread.currentThread().getContextClassLoader()
+          .getResource(allocFilePath);
+      if (url == null) {
+        LOG.warn(allocFilePath + " not found on the classpath.");
+        allocFile = null;
+      } else if (!url.getProtocol().equalsIgnoreCase("file")) {
+        throw new RuntimeException("Allocation file " + url
+            + " found on the classpath is not on the local filesystem.");
+      } else {
+        allocFile = new File(url.getPath());
+      }
+    }
+    return allocFile;
   }
 
   public String getEventlogDir() {

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java?rev=1528203&r1=1528202&r2=1528203&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
Tue Oct  1 20:16:30 2013
@@ -75,11 +75,8 @@ public class QueueManager {
 
   private final FairScheduler scheduler;
 
-  private Object allocFile; // Path to XML file containing allocations. This
-                            // is either a URL to specify a classpath resource
-                            // (if the fair-scheduler.xml on the classpath is
-                            // used) or a String to specify an absolute path (if
-                            // mapred.fairscheduler.allocation.file is used).
+  // Path to XML file containing allocations. 
+  private File allocFile; 
 
   private final Collection<FSLeafQueue> leafQueues = 
       new CopyOnWriteArrayList<FSLeafQueue>();
@@ -107,16 +104,7 @@ public class QueueManager {
     queues.put(rootQueue.getName(), rootQueue);
     
     this.allocFile = conf.getAllocationFile();
-    if (allocFile == null) {
-      // No allocation file specified in jobconf. Use the default allocation
-      // file, fair-scheduler.xml, looking for it on the classpath.
-      allocFile = new Configuration().getResource("fair-scheduler.xml");
-      if (allocFile == null) {
-        LOG.error("The fair scheduler allocation file fair-scheduler.xml was "
-            + "not found on the classpath, and no other config file is given "
-            + "through mapred.fairscheduler.allocation.file.");
-      }
-    }
+    
     reloadAllocs();
     lastSuccessfulReload = scheduler.getClock().getTime();
     lastReloadAttempt = scheduler.getClock().getTime();
@@ -255,14 +243,7 @@ public class QueueManager {
       try {
         // Get last modified time of alloc file depending whether it's a String
         // (for a path name) or an URL (for a classloader resource)
-        long lastModified;
-        if (allocFile instanceof String) {
-          File file = new File((String) allocFile);
-          lastModified = file.lastModified();
-        } else { // allocFile is an URL
-          URLConnection conn = ((URL) allocFile).openConnection();
-          lastModified = conn.getLastModified();
-        }
+        long lastModified = allocFile.lastModified();
         if (lastModified > lastSuccessfulReload &&
             time > lastModified + ALLOC_RELOAD_WAIT) {
           reloadAllocs();
@@ -321,12 +302,7 @@ public class QueueManager {
       DocumentBuilderFactory.newInstance();
     docBuilderFactory.setIgnoringComments(true);
     DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
-    Document doc;
-    if (allocFile instanceof String) {
-      doc = builder.parse(new File((String) allocFile));
-    } else {
-      doc = builder.parse(allocFile.toString());
-    }
+    Document doc = builder.parse(allocFile);
     Element root = doc.getDocumentElement();
     if (!"allocations".equals(root.getTagName()))
       throw new AllocationConfigurationException("Bad fair scheduler config " +

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java?rev=1528203&r1=1528202&r2=1528203&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
Tue Oct  1 20:16:30 2013
@@ -20,6 +20,11 @@ package org.apache.hadoop.yarn.server.re
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration.parseResourceConfigValue;
 import static org.junit.Assert.assertEquals;
 
+import java.io.File;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.junit.Test;
 
@@ -55,4 +60,15 @@ public class TestFairSchedulerConfigurat
   public void testGibberish() throws Exception {
     parseResourceConfigValue("1o24vc0res");
   }
+  
+  @Test
+  public void testGetAllocationFileFromClasspath() {
+    FairSchedulerConfiguration conf = new FairSchedulerConfiguration(
+        new Configuration());
+    conf.set(FairSchedulerConfiguration.ALLOCATION_FILE,
+        "test-fair-scheduler.xml");
+    File allocationFile = conf.getAllocationFile();
+    Assert.assertEquals("test-fair-scheduler.xml", allocationFile.getName());
+    Assert.assertTrue(allocationFile.exists());
+  }
 }

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1528203&r1=1528202&r2=1528203&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
(original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
Tue Oct  1 20:16:30 2013
@@ -81,7 +81,7 @@ Hadoop MapReduce Next Generation - Fair 
   distribute the resources assigned to them to their children in the same
   fashion.  Applications may only be scheduled on leaf queues. Queues can be
   specified as children of other queues by placing them as sub-elements of 
-  their parents in the fair scheduler configuration file.
+  their parents in the fair scheduler allocation file.
   
   A queue's name starts with the names of its parents, with periods as
   separators. So a queue named "queue1" under the root queue, would be referred
@@ -118,16 +118,20 @@ Hadoop MapReduce Next Generation - Fair 
   Customizing the Fair Scheduler typically involves altering two files. First, 
   scheduler-wide options can be set by adding configuration properties in the 
   yarn-site.xml file in your existing configuration directory. Second, in 
-  most cases users will want to create a manifest file listing which queues 
-  exist and their respective weights and capacities. The location of this file 
-  is flexible - but it must be declared in yarn-site.xml.
+  most cases users will want to create an allocation file listing which queues 
+  exist and their respective weights and capacities. The allocation file
+  is reloaded every 10 seconds, allowing changes to be made on the fly.
+
+Properties that can be placed in yarn-site.xml
 
  * <<<yarn.scheduler.fair.allocation.file>>>
 
    * Path to allocation file. An allocation file is an XML manifest describing
      queues and their properties, in addition to certain policy defaults. This file
-     must be in XML format as described in the next section.
-     Defaults to fair-scheduler.xml in configuration directory.
+     must be in the XML format described in the next section. If a relative path is
+     given, the file is searched for on the classpath (which typically includes
+     the Hadoop conf directory).
+     Defaults to fair-scheduler.xml.
 
  * <<<yarn.scheduler.fair.user-as-default-queue>>>
 



Mime
View raw message