hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject [18/23] hadoop git commit: YARN-4537. Pull out priority comparison from fifocomparator and use compound comparator for FifoOrdering policy. Contributed by Rohith Sharma K S
Date Tue, 12 Jan 2016 04:32:17 GMT
YARN-4537. Pull out priority comparison from fifocomparator and use compound comparator for
FifoOrdering policy. Contributed by Rohith Sharma K S


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b8942be8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b8942be8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b8942be8

Branch: refs/heads/HDFS-1312
Commit: b8942be888e1f6f7049398f3d0322602a58ee312
Parents: de37f37
Author: Jian He <jianhe@apache.org>
Authored: Mon Jan 11 16:44:28 2016 -0800
Committer: Jian He <jianhe@apache.org>
Committed: Mon Jan 11 16:44:28 2016 -0800

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 ++
 .../dev-support/findbugs-exclude.xml            |  4 +++
 .../scheduler/policy/FifoComparator.java        |  7 -----
 .../scheduler/policy/FifoOrderingPolicy.java    |  8 +++--
 .../policy/TestFifoOrderingPolicy.java          | 33 +++++++++++++++++---
 5 files changed, 41 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8942be8/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e0c81b0..e58f15b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -683,6 +683,9 @@ Release 2.8.0 - UNRELEASED
     YARN-4544. All the log messages about rolling monitoring interval are
     shown with WARN level. (Takashi Ohnishi via aajisaka)
 
+    YARN-4537. Pull out priority comparison from fifocomparator and use compound
+    comparator for FifoOrdering policy. (Rohith Sharma K S via jianhe)
+
   OPTIMIZATIONS
 
     YARN-3339. TestDockerContainerExecutor should pull a single image and not

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8942be8/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index c79a35e..6fb3945 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -159,6 +159,10 @@
     <Bug pattern="SE_COMPARATOR_SHOULD_BE_SERIALIZABLE" />
   </Match>
   <Match>
+    <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.PriorityComparator"
/>
+    <Bug pattern="SE_COMPARATOR_SHOULD_BE_SERIALIZABLE" />
+  </Match>
+  <Match>
     <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PartitionedQueueComparator"
/>
     <Bug pattern="SE_COMPARATOR_SHOULD_BE_SERIALIZABLE" />
   </Match>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8942be8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java
index 1045386..112c50f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java
@@ -20,9 +20,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy;
 
 import java.util.*;
 
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.*;
-
 /**
  * A Comparator which orders SchedulableEntities by input order
  */
@@ -31,10 +28,6 @@ public class FifoComparator
       
     @Override
   public int compare(SchedulableEntity r1, SchedulableEntity r2) {
-    if (r1.getPriority() != null
-        && !r1.getPriority().equals(r2.getPriority())) {
-      return r1.getPriority().compareTo(r2.getPriority());
-    }
     int res = r1.compareInputOrderTo(r2);
     return res;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8942be8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoOrderingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoOrderingPolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoOrderingPolicy.java
index 74a422c..10f8eeb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoOrderingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoOrderingPolicy.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy;
 
 import java.util.*;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.*;
 
 /**
  * An OrderingPolicy which orders SchedulableEntities by input order
@@ -28,8 +27,13 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.*;
 public class FifoOrderingPolicy<S extends SchedulableEntity> extends AbstractComparatorOrderingPolicy<S>
{
   
   public FifoOrderingPolicy() {
-    this.comparator = new FifoComparator();
+    List<Comparator<SchedulableEntity>> comparators =
+        new ArrayList<Comparator<SchedulableEntity>>();
+    comparators.add(new PriorityComparator());
+    comparators.add(new FifoComparator());
+    this.comparator = new CompoundComparator(comparators);
     this.schedulableEntities = new TreeSet<S>(comparator);
+
   }
   
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8942be8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFifoOrderingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFifoOrderingPolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFifoOrderingPolicy.java
index e05ef26..447618f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFifoOrderingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFifoOrderingPolicy.java
@@ -24,11 +24,6 @@ import org.junit.Assert;
 import org.junit.Test;
 
 import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.util.resource.Resources;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
-
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
 
 public class TestFifoOrderingPolicy {
   
@@ -80,4 +75,32 @@ public class TestFifoOrderingPolicy {
     }
   }
   
+  @Test
+  public void testFifoOrderingPolicyAlongWithPriorty() {
+    FifoOrderingPolicy<MockSchedulableEntity> policy =
+        new FifoOrderingPolicy<MockSchedulableEntity>();
+    MockSchedulableEntity r1 = new MockSchedulableEntity();
+    MockSchedulableEntity r2 = new MockSchedulableEntity();
+
+    Priority p1 = Priority.newInstance(1);
+    Priority p2 = Priority.newInstance(0);
+
+    // Both r1 and r1 priority is null
+    Assert.assertEquals(0, policy.getComparator().compare(r1, r2));
+
+    // r1 is null and r2 is not null
+    r2.setApplicationPriority(p2);
+    Assert.assertEquals(-1, policy.getComparator().compare(r1, r2));
+
+    // r1 is not null and r2 is null
+    r2.setApplicationPriority(null);
+    r1.setApplicationPriority(p1);
+    Assert.assertEquals(1, policy.getComparator().compare(r1, r2));
+
+    // r1 is not null and r2 is not null
+    r1.setApplicationPriority(p1);
+    r2.setApplicationPriority(p2);
+    Assert.assertEquals(-1, policy.getComparator().compare(r1, r2));
+  }
+
 }


Mime
View raw message