hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ka...@apache.org
Subject hadoop git commit: YARN-5616. Clean up WeightAdjuster. (Yufei Gu via kasha)
Date Wed, 07 Sep 2016 05:40:32 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 680be58aa -> c0e492e50


YARN-5616. Clean up WeightAdjuster. (Yufei Gu via kasha)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c0e492e5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c0e492e5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c0e492e5

Branch: refs/heads/trunk
Commit: c0e492e50fa98d423c8a61c94d2d1f9553558b6d
Parents: 680be58
Author: Karthik Kambatla <kasha@apache.org>
Authored: Tue Sep 6 22:40:20 2016 -0700
Committer: Karthik Kambatla <kasha@apache.org>
Committed: Tue Sep 6 22:40:20 2016 -0700

----------------------------------------------------------------------
 .../scheduler/fair/FairScheduler.java           |  5 --
 .../scheduler/fair/NewAppWeightBooster.java     | 60 --------------------
 .../scheduler/fair/WeightAdjuster.java          | 36 ------------
 3 files changed, 101 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0e492e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index ac8dbe1..15e2318 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -186,7 +186,6 @@ public class FairScheduler extends
                                       // an app can be reserved on
 
   protected boolean sizeBasedWeight; // Give larger weights to larger jobs
-  protected WeightAdjuster weightAdjuster; // Can be null for no weight adjuster
   protected boolean continuousSchedulingEnabled; // Continuous Scheduling enabled or not
   protected int continuousSchedulingSleepMs; // Sleep time for each pass in continuous scheduling
   private Comparator<FSSchedulerNode> nodeAvailableResourceComparator =
@@ -563,10 +562,6 @@ public class FairScheduler extends
       weight = Math.log1p(app.getDemand().getMemorySize()) / Math.log(2);
     }
     weight *= app.getPriority().getPriority();
-    if (weightAdjuster != null) {
-      // Run weight through the user-supplied weightAdjuster
-      weight = weightAdjuster.adjustWeight(app, weight);
-    }
     ResourceWeights resourceWeights = app.getResourceWeights();
     resourceWeights.setWeight((float)weight);
     return resourceWeights;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0e492e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java
deleted file mode 100644
index fb32e56..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-
-/**
- * A {@link WeightAdjuster} implementation that gives a weight boost to new jobs
- * for a certain amount of time -- by default, a 3x weight boost for 60 seconds.
- * This can be used to make shorter jobs finish faster, emulating Shortest Job
- * First scheduling while not starving long jobs.
- */
-@Private
-@Unstable
-public class NewAppWeightBooster extends Configured implements WeightAdjuster {
-  private static final float DEFAULT_FACTOR = 3;
-  private static final long DEFAULT_DURATION = 5 * 60 * 1000;
-
-  private float factor;
-  private long duration;
-
-  public void setConf(Configuration conf) {
-    if (conf != null) {
-      factor = conf.getFloat("mapred.newjobweightbooster.factor",
-          DEFAULT_FACTOR);
-      duration = conf.getLong("mapred.newjobweightbooster.duration",
-          DEFAULT_DURATION);
-    }
-    super.setConf(conf);
-  }
-
-  public double adjustWeight(FSAppAttempt app, double curWeight) {
-    long start = app.getStartTime();
-    long now = System.currentTimeMillis();
-    if (now - start < duration) {
-      return curWeight * factor;
-    } else {
-      return curWeight;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0e492e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java
deleted file mode 100644
index 67364ed..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configurable;
-
-/**
- * A pluggable object for altering the weights of apps in the fair scheduler,
- * which is used for example by {@link NewAppWeightBooster} to give higher
- * weight to new jobs so that short jobs finish faster.
- *
- * May implement {@link Configurable} to access configuration parameters.
- */
-@Private
-@Unstable
-public interface WeightAdjuster {
-  public double adjustWeight(FSAppAttempt app, double curWeight);
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message