hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jh...@apache.org
Subject [30/50] [abbrv] hadoop git commit: YARN-2162. Add ability in Fair Scheduler to optionally configure maxResources in terms of percentage(addendum). (Yufei Gu)
Date Sat, 07 Oct 2017 00:06:34 GMT
YARN-2162. Add ability in Fair Scheduler to optionally configure maxResources in terms of percentage(addendum).
(Yufei Gu)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/49ae5381
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/49ae5381
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/49ae5381

Branch: refs/heads/YARN-5734
Commit: 49ae538164bf631ebf961006a06c8ba0f3469b89
Parents: 99292ad
Author: Yufei Gu <yufei@apache.org>
Authored: Fri Oct 6 00:08:25 2017 -0700
Committer: Yufei Gu <yufei@apache.org>
Committed: Fri Oct 6 00:08:25 2017 -0700

----------------------------------------------------------------------
 .../scheduler/fair/ConfigurableResource.java    | 72 ++++++++++++++++++++
 .../fair/TestConfigurableResource.java          | 68 ++++++++++++++++++
 2 files changed, 140 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/49ae5381/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java
new file mode 100644
index 0000000..8de41be
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.Resource;
+
+/**
+ * A {@code ConfigurableResource} object represents an entity that is used to
+ * configure resources, such as maximum resources of a queue. It can be
+ * percentage of cluster resources or an absolute value.
+ */
+@Private
+@Unstable
+public class ConfigurableResource {
+  private final Resource resource;
+  private final double[] percentages;
+
+  public ConfigurableResource(double[] percentages) {
+    this.percentages = percentages.clone();
+    this.resource = null;
+  }
+
+  public ConfigurableResource(Resource resource) {
+    this.percentages = null;
+    this.resource = resource;
+  }
+
+  /**
+   * Get resource by multiplying the cluster resource and the percentage of
+   * each resource respectively. Return the absolute resource if either
+   * {@code percentages} or {@code clusterResource) is null.
+   *
+   * @param clusterResource the cluster resource
+   * @return resource
+   */
+  public Resource getResource(Resource clusterResource) {
+    if (percentages != null && clusterResource != null) {
+      long memory = (long) (clusterResource.getMemorySize() * percentages[0]);
+      int vcore = (int) (clusterResource.getVirtualCores() * percentages[1]);
+      return Resource.newInstance(memory, vcore);
+    } else {
+      return resource;
+    }
+  }
+
+  /**
+   * Get the absolute resource.
+   *
+   * @return the absolute resource
+   */
+  public Resource getResource() {
+    return resource;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/49ae5381/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestConfigurableResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestConfigurableResource.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestConfigurableResource.java
new file mode 100644
index 0000000..249d1f7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestConfigurableResource.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+/**
+ * To test class {@link ConfigurableResource}.
+ */
+public class TestConfigurableResource {
+  private final Resource clusterResource = Resources.createResource(2048, 2);
+
+  @Test
+  public void testGetResourceWithPercentage() {
+    ConfigurableResource configurableResource =
+        new ConfigurableResource(new double[] {0.5, 0.5});
+    assertEquals(
+        configurableResource.getResource(clusterResource).getMemorySize(),
+        1024);
+    assertEquals(
+        configurableResource.getResource(clusterResource).getVirtualCores(), 1);
+
+    assertNull("The absolute resource should be null since object"
+            + " configurableResource is initialized with percentages",
+        configurableResource.getResource());
+    assertNull("The absolute resource should be null since cluster resource"
+        + " is null", configurableResource.getResource(null));
+  }
+
+  @Test
+  public void testGetResourceWithAbsolute() {
+    ConfigurableResource configurableResource =
+        new ConfigurableResource(Resources.createResource(3072, 3));
+    assertEquals(configurableResource.getResource().getMemorySize(), 3072);
+    assertEquals(configurableResource.getResource().getVirtualCores(), 3);
+
+    assertEquals(
+        configurableResource.getResource(clusterResource).getMemorySize(),
+        3072);
+    assertEquals(
+        configurableResource.getResource(clusterResource).getVirtualCores(),
+        3);
+
+    assertEquals(configurableResource.getResource(null).getMemorySize(), 3072);
+    assertEquals(configurableResource.getResource(null).getVirtualCores(), 3);
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message