Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id C6213200B81 for ; Tue, 13 Sep 2016 10:29:22 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id C4C28160AD8; Tue, 13 Sep 2016 08:29:22 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id E7938160AD3 for ; Tue, 13 Sep 2016 10:29:21 +0200 (CEST) Received: (qmail 62529 invoked by uid 500); 13 Sep 2016 08:29:16 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 61772 invoked by uid 99); 13 Sep 2016 08:29:16 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 13 Sep 2016 08:29:16 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id F1576DFD9F; Tue, 13 Sep 2016 08:29:15 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: drankye@apache.org To: common-commits@hadoop.apache.org Date: Tue, 13 Sep 2016 08:29:34 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [20/50] [abbrv] hadoop git commit: YARN-5616. Clean up WeightAdjuster. (Yufei Gu via kasha) archived-at: Tue, 13 Sep 2016 08:29:23 -0000 YARN-5616. Clean up WeightAdjuster. (Yufei Gu via kasha) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c0e492e5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c0e492e5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c0e492e5 Branch: refs/heads/HDFS-10285 Commit: c0e492e50fa98d423c8a61c94d2d1f9553558b6d Parents: 680be58 Author: Karthik Kambatla Authored: Tue Sep 6 22:40:20 2016 -0700 Committer: Karthik Kambatla Committed: Tue Sep 6 22:40:20 2016 -0700 ---------------------------------------------------------------------- .../scheduler/fair/FairScheduler.java | 5 -- .../scheduler/fair/NewAppWeightBooster.java | 60 -------------------- .../scheduler/fair/WeightAdjuster.java | 36 ------------ 3 files changed, 101 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0e492e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index ac8dbe1..15e2318 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -186,7 +186,6 @@ public class FairScheduler extends // an app can be reserved on protected boolean sizeBasedWeight; // Give larger weights to larger jobs - protected WeightAdjuster weightAdjuster; // Can be null for no weight adjuster protected boolean continuousSchedulingEnabled; // Continuous Scheduling enabled or not protected int continuousSchedulingSleepMs; // Sleep time for each pass in continuous scheduling private Comparator nodeAvailableResourceComparator = @@ -563,10 +562,6 @@ public class FairScheduler extends weight = Math.log1p(app.getDemand().getMemorySize()) / Math.log(2); } weight *= app.getPriority().getPriority(); - if (weightAdjuster != null) { - // Run weight through the user-supplied weightAdjuster - weight = weightAdjuster.adjustWeight(app, weight); - } ResourceWeights resourceWeights = app.getResourceWeights(); resourceWeights.setWeight((float)weight); return resourceWeights; http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0e492e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java deleted file mode 100644 index fb32e56..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; - -/** - * A {@link WeightAdjuster} implementation that gives a weight boost to new jobs - * for a certain amount of time -- by default, a 3x weight boost for 60 seconds. - * This can be used to make shorter jobs finish faster, emulating Shortest Job - * First scheduling while not starving long jobs. - */ -@Private -@Unstable -public class NewAppWeightBooster extends Configured implements WeightAdjuster { - private static final float DEFAULT_FACTOR = 3; - private static final long DEFAULT_DURATION = 5 * 60 * 1000; - - private float factor; - private long duration; - - public void setConf(Configuration conf) { - if (conf != null) { - factor = conf.getFloat("mapred.newjobweightbooster.factor", - DEFAULT_FACTOR); - duration = conf.getLong("mapred.newjobweightbooster.duration", - DEFAULT_DURATION); - } - super.setConf(conf); - } - - public double adjustWeight(FSAppAttempt app, double curWeight) { - long start = app.getStartTime(); - long now = System.currentTimeMillis(); - if (now - start < duration) { - return curWeight * factor; - } else { - return curWeight; - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0e492e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java deleted file mode 100644 index 67364ed..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.conf.Configurable; - -/** - * A pluggable object for altering the weights of apps in the fair scheduler, - * which is used for example by {@link NewAppWeightBooster} to give higher - * weight to new jobs so that short jobs finish faster. - * - * May implement {@link Configurable} to access configuration parameters. - */ -@Private -@Unstable -public interface WeightAdjuster { - public double adjustWeight(FSAppAttempt app, double curWeight); -} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org