hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jh...@apache.org
Subject [10/38] hadoop git commit: YARN-5703. ReservationAgents are not correctly configured. Contributed by Manikandan R.
Date Fri, 03 Mar 2017 01:57:18 GMT
YARN-5703. ReservationAgents are not correctly configured. Contributed by Manikandan R.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f5b031d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f5b031d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f5b031d

Branch: refs/heads/YARN-5734
Commit: 5f5b031d1f20cb7f621db41979e963eaa42cf52f
Parents: 4d33683
Author: Naganarasimha <naganarasimha_gr@apache.org>
Authored: Mon Feb 27 20:38:29 2017 +0530
Committer: Naganarasimha <naganarasimha_gr@apache.org>
Committed: Mon Feb 27 20:38:29 2017 +0530

----------------------------------------------------------------------
 .../reservation/AbstractReservationSystem.java   |  8 ++++++--
 .../planning/AlignedPlannerWithGreedy.java       | 16 +++++++++-------
 .../planning/GreedyReservationAgent.java         | 19 ++++++-------------
 .../reservation/planning/PlanningAlgorithm.java  |  4 ++++
 .../reservation/planning/ReservationAgent.java   |  8 ++++++++
 .../planning/TryManyReservationAgents.java       |  5 ++++-
 .../reservation/planning/TestAlignedPlanner.java |  3 +++
 .../planning/TestGreedyReservationAgent.java     |  4 ++--
 8 files changed, 42 insertions(+), 25 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
index 8769ca1..5ef4912 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
@@ -451,12 +451,16 @@ public abstract class AbstractReservationSystem extends AbstractService
     try {
       Class<?> agentClazz = conf.getClassByName(agentClassName);
       if (ReservationAgent.class.isAssignableFrom(agentClazz)) {
-        return (ReservationAgent) ReflectionUtils.newInstance(agentClazz, conf);
+        ReservationAgent resevertionAgent =
+            (ReservationAgent) agentClazz.newInstance();
+        resevertionAgent.init(conf);
+        return resevertionAgent;
       } else {
         throw new YarnRuntimeException("Class: " + agentClassName
             + " not instance of " + ReservationAgent.class.getCanonicalName());
       }
-    } catch (ClassNotFoundException e) {
+    } catch (ClassNotFoundException | InstantiationException
+        | IllegalAccessException e) {
       throw new YarnRuntimeException("Could not instantiate Agent: "
           + agentClassName + " for queue: " + queueName, e);
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/AlignedPlannerWithGreedy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/AlignedPlannerWithGreedy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/AlignedPlannerWithGreedy.java
index b23cf1e..00c2333 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/AlignedPlannerWithGreedy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/AlignedPlannerWithGreedy.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning;
 import java.util.LinkedList;
 import java.util.List;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ReservationDefinition;
 import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
@@ -35,22 +36,25 @@ import org.slf4j.LoggerFactory;
 public class AlignedPlannerWithGreedy implements ReservationAgent {
 
   // Default smoothness factor
-  private static final int DEFAULT_SMOOTHNESS_FACTOR = 10;
+  public static final int DEFAULT_SMOOTHNESS_FACTOR = 10;
+  public static final String SMOOTHNESS_FACTOR =
+      "yarn.resourcemanager.reservation-system.smoothness-factor";
 
   // Log
   private static final Logger LOG = LoggerFactory
       .getLogger(AlignedPlannerWithGreedy.class);
 
   // Smoothness factor
-  private final ReservationAgent planner;
+  private ReservationAgent planner;
 
   // Constructor
   public AlignedPlannerWithGreedy() {
-    this(DEFAULT_SMOOTHNESS_FACTOR);
   }
 
-  // Constructor
-  public AlignedPlannerWithGreedy(int smoothnessFactor) {
+  @Override
+  public void init(Configuration conf) {
+    int smoothnessFactor =
+        conf.getInt(SMOOTHNESS_FACTOR, DEFAULT_SMOOTHNESS_FACTOR);
 
     // List of algorithms
     List<ReservationAgent> listAlg = new LinkedList<ReservationAgent>();
@@ -71,7 +75,6 @@ public class AlignedPlannerWithGreedy implements ReservationAgent {
     // 1. Attempt to execute algAligned
     // 2. If failed, fall back to algGreedy
     planner = new TryManyReservationAgents(listAlg);
-
   }
 
   @Override
@@ -119,5 +122,4 @@ public class AlignedPlannerWithGreedy implements ReservationAgent {
     return planner.deleteReservation(reservationId, user, plan);
 
   }
-
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/GreedyReservationAgent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/GreedyReservationAgent.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/GreedyReservationAgent.java
index 915a834..1559b97 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/GreedyReservationAgent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/GreedyReservationAgent.java
@@ -46,25 +46,19 @@ public class GreedyReservationAgent implements ReservationAgent {
       .getLogger(GreedyReservationAgent.class);
 
   // Greedy planner
-  private final ReservationAgent planner;
-
+  private ReservationAgent planner;
   public final static String GREEDY_FAVOR_EARLY_ALLOCATION =
       "yarn.resourcemanager.reservation-system.favor-early-allocation";
-
   public final static boolean DEFAULT_GREEDY_FAVOR_EARLY_ALLOCATION = true;
-
-  private final boolean allocateLeft;
+  private boolean allocateLeft;
 
   public GreedyReservationAgent() {
-    this(new Configuration());
   }
 
-  public GreedyReservationAgent(Configuration yarnConfiguration) {
-
-    allocateLeft =
-        yarnConfiguration.getBoolean(GREEDY_FAVOR_EARLY_ALLOCATION,
-            DEFAULT_GREEDY_FAVOR_EARLY_ALLOCATION);
-
+  @Override
+  public void init(Configuration conf) {
+    allocateLeft = conf.getBoolean(GREEDY_FAVOR_EARLY_ALLOCATION,
+        DEFAULT_GREEDY_FAVOR_EARLY_ALLOCATION);
     if (allocateLeft) {
       LOG.info("Initializing the GreedyReservationAgent to favor \"early\""
           + " (left) allocations (controlled by parameter: "
@@ -78,7 +72,6 @@ public class GreedyReservationAgent implements ReservationAgent {
     planner =
         new IterativePlanner(new StageEarliestStartByJobArrival(),
             new StageAllocatorGreedyRLE(allocateLeft), allocateLeft);
-
   }
 
   public boolean isAllocateLeft(){

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
index e1b508d..199bfa5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning;
 import java.util.Map;
 import java.util.Map.Entry;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ReservationDefinition;
 import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.api.records.Resource;
@@ -206,4 +207,7 @@ public abstract class PlanningAlgorithm implements ReservationAgent {
 
   }
 
+  @Override
+  public void init(Configuration conf) {
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/ReservationAgent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/ReservationAgent.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/ReservationAgent.java
index bdea2f4..52e7055 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/ReservationAgent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/ReservationAgent.java
@@ -17,6 +17,7 @@
  *******************************************************************************/
 package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ReservationDefinition;
 import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
@@ -70,4 +71,11 @@ public interface ReservationAgent {
   public boolean deleteReservation(ReservationId reservationId, String user,
       Plan plan) throws PlanningException;
 
+  /**
+   * Init configuration.
+   *
+   * @param conf Configuration
+   */
+  void init(Configuration conf);
+
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TryManyReservationAgents.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TryManyReservationAgents.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TryManyReservationAgents.java
index 1d37ce5..ab2e1e0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TryManyReservationAgents.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TryManyReservationAgents.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning;
 import java.util.LinkedList;
 import java.util.List;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ReservationDefinition;
 import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
@@ -110,5 +111,7 @@ public class TryManyReservationAgents implements ReservationAgent {
     return plan.deleteReservation(reservationId);
 
   }
-
+  @Override
+  public void init(Configuration conf) {
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java
index 4b01eb9..2645366 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java
@@ -726,8 +726,11 @@ public class TestAlignedPlanner {
     QueueMetrics queueMetrics = mock(QueueMetrics.class);
     RMContext context = ReservationSystemTestUtil.createMockRMContext();
 
+    conf.setInt(AlignedPlannerWithGreedy.SMOOTHNESS_FACTOR,
+        AlignedPlannerWithGreedy.DEFAULT_SMOOTHNESS_FACTOR);
     // Set planning agent
     agent = new AlignedPlannerWithGreedy();
+    agent.init(conf);
 
     // Create Plan
     plan =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java
index ec11ffe..6d1cfa8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java
@@ -110,8 +110,8 @@ public class TestGreedyReservationAgent {
     // setting conf to
     conf.setBoolean(GreedyReservationAgent.GREEDY_FAVOR_EARLY_ALLOCATION,
         allocateLeft);
-
-    agent = new GreedyReservationAgent(conf);
+    agent = new GreedyReservationAgent();
+    agent.init(conf);
 
     QueueMetrics queueMetrics = mock(QueueMetrics.class);
     RMContext context = ReservationSystemTestUtil.createMockRMContext();


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message