hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bra...@apache.org
Subject [32/49] hadoop git commit: YARN-8877. [CSI] Extend service spec to allow setting resource attributes. Contributed by Weiwei Yang.
Date Tue, 13 Nov 2018 07:50:59 GMT
YARN-8877. [CSI] Extend service spec to allow setting resource attributes. Contributed by Weiwei
Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/42f3a708
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/42f3a708
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/42f3a708

Branch: refs/heads/HDFS-13891
Commit: 42f3a7082a90bc71f0e86dc1e50b0c77b05489cf
Parents: 3c9d97b
Author: Sunil G <sunilg@apache.org>
Authored: Mon Nov 12 21:09:30 2018 +0530
Committer: Sunil G <sunilg@apache.org>
Committed: Mon Nov 12 21:09:30 2018 +0530

----------------------------------------------------------------------
 .../yarn/api/records/ResourceInformation.java   |  7 +++
 .../api/records/ResourceInformation.java        | 18 ++++++++
 .../yarn/service/component/Component.java       |  3 +-
 .../hadoop/yarn/service/TestServiceAM.java      | 47 ++++++++++++++++++++
 .../yarn/service/conf/ExampleAppJson.java       |  1 +
 .../yarn/service/conf/TestAppJsonResolve.java   | 18 ++++++++
 .../yarn/service/conf/examples/external3.json   | 26 +++++++++++
 .../markdown/yarn-service/YarnServiceAPI.md     |  2 +-
 8 files changed, 120 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f3a708/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
index 057e94e..047c09a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
@@ -276,6 +276,13 @@ public class ResourceInformation implements Comparable<ResourceInformation>
{
   }
 
   public static ResourceInformation newInstance(String name, String units,
+      long value, Map<String, String> attributes) {
+    return ResourceInformation
+        .newInstance(name, units, value, ResourceTypes.COUNTABLE, 0L,
+            Long.MAX_VALUE, null, attributes);
+  }
+
+  public static ResourceInformation newInstance(String name, String units,
       ResourceTypes resourceType) {
     return ResourceInformation.newInstance(name, units, 0L, resourceType, 0L,
         Long.MAX_VALUE);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f3a708/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ResourceInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ResourceInformation.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ResourceInformation.java
index 103fffb..e466ce7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ResourceInformation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ResourceInformation.java
@@ -18,10 +18,12 @@
 package org.apache.hadoop.yarn.service.api.records;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.ImmutableMap;
 import com.google.gson.annotations.SerializedName;
 import io.swagger.annotations.ApiModel;
 import io.swagger.annotations.ApiModelProperty;
 
+import java.util.Map;
 import java.util.Objects;
 
 /**
@@ -35,11 +37,25 @@ public class ResourceInformation {
   @SerializedName("unit")
   private String unit = null;
 
+  @SerializedName("attributes")
+  private Map<String, String> attributes = null;
+
   public ResourceInformation value(Long value) {
     this.value = value;
     return this;
   }
 
+  @ApiModelProperty(value = "")
+  @JsonProperty("attributes")
+  public Map<String, String> getAttributes() {
+    return attributes == null ? ImmutableMap.of() : attributes;
+  }
+
+  public ResourceInformation attributes(Map<String, String> attributes) {
+    this.attributes = attributes;
+    return this;
+  }
+
   /**
    * Integer value of the resource.
    *
@@ -98,6 +114,8 @@ public class ResourceInformation {
     sb.append("class ResourceInformation {\n");
     sb.append("    value: ").append(toIndentedString(value)).append("\n");
     sb.append("    unit: ").append(toIndentedString(unit)).append("\n");
+    sb.append("    attributes: ").append(toIndentedString(attributes))
+        .append("\n");
     sb.append("}");
     return sb.toString();
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f3a708/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
index 526bde0..7833b06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -701,7 +701,8 @@ public class Component implements EventHandler<ComponentEvent> {
             org.apache.hadoop.yarn.api.records.ResourceInformation.newInstance(
                 entry.getKey(),
                 specInfo.getUnit(),
-                specInfo.getValue());
+                specInfo.getValue(),
+                specInfo.getAttributes());
         resource.setResourceInformation(resourceName, ri);
       }
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f3a708/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java
index 80f4910..51c27e8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java
@@ -456,4 +456,51 @@ public class TestServiceAM extends ServiceTestUtils{
       Assert.fail("Fail to sync sysfs.");
     }
   }
+
+  @Test
+  public void testScheduleWithResourceAttributes() throws Exception {
+    ApplicationId applicationId = ApplicationId.newInstance(123456, 1);
+    Service exampleApp = new Service();
+    exampleApp.setId(applicationId.toString());
+    exampleApp.setName("testScheduleWithResourceAttributes");
+    exampleApp.setVersion("v1");
+
+    List<ResourceTypeInfo> resourceTypeInfos = new ArrayList<>(
+        ResourceUtils.getResourcesTypeInfo());
+    // Add 3rd resource type.
+    resourceTypeInfos.add(ResourceTypeInfo
+        .newInstance("test-resource", "", ResourceTypes.COUNTABLE));
+    // Reinitialize resource types
+    ResourceUtils.reinitializeResources(resourceTypeInfos);
+
+    Component serviceCompoent = createComponent("compa", 1, "pwd");
+    serviceCompoent.getResource().setResourceInformations(
+        ImmutableMap.of("test-resource",
+            new ResourceInformation()
+                .value(1234L)
+                .unit("Gi")
+                .attributes(ImmutableMap.of("k1", "v1", "k2", "v2"))));
+    exampleApp.addComponent(serviceCompoent);
+
+    MockServiceAM am = new MockServiceAM(exampleApp);
+    am.init(conf);
+    am.start();
+
+    ServiceScheduler serviceScheduler = am.context.scheduler;
+    AMRMClientAsync<AMRMClient.ContainerRequest> amrmClientAsync =
+        serviceScheduler.getAmRMClient();
+
+    Collection<AMRMClient.ContainerRequest> rr =
+        amrmClientAsync.getMatchingRequests(0);
+    Assert.assertEquals(1, rr.size());
+
+    org.apache.hadoop.yarn.api.records.Resource capability =
+        rr.iterator().next().getCapability();
+    Assert.assertEquals(1234L, capability.getResourceValue("test-resource"));
+    Assert.assertEquals("Gi",
+        capability.getResourceInformation("test-resource").getUnits());
+    Assert.assertEquals(2, capability.getResourceInformation("test-resource")
+        .getAttributes().size());
+    am.stop();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f3a708/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
index 5fdd2ab..754b589 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
@@ -38,6 +38,7 @@ public final class ExampleAppJson {
   public static final String EXTERNAL_JSON_0 = "external0.json";
   public static final String EXTERNAL_JSON_1 = "external1.json";
   public static final String EXTERNAL_JSON_2 = "external2.json";
+  public static final String EXTERNAL_JSON_3 = "external3.json";
 
   public static final String PACKAGE = "/org/apache/hadoop/yarn/service/conf/examples/";
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f3a708/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
index 73f7fa1..25c502f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
@@ -20,7 +20,9 @@ package org.apache.hadoop.yarn.service.conf;
 
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.service.ServiceTestUtils;
+import org.apache.hadoop.yarn.service.api.records.Component;
 import org.apache.hadoop.yarn.service.api.records.Resource;
+import org.apache.hadoop.yarn.service.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.service.api.records.Service;
 import org.apache.hadoop.yarn.service.api.records.ConfigFile;
 import org.apache.hadoop.yarn.service.api.records.Configuration;
@@ -214,4 +216,20 @@ public class TestAppJsonResolve extends Assert {
     other = orig.getComponent("other").getConfiguration();
     assertEquals(0, other.getProperties().size());
   }
+
+  @Test
+  public void testSetResourceAttributes() throws IOException {
+    Service orig = ExampleAppJson.loadResource(EXTERNAL_JSON_3);
+    Component component = orig.getComponent("volume-service");
+    Assert.assertNotNull(component);
+    Map<String, ResourceInformation> adResource = component
+        .getResource().getAdditional();
+    Assert.assertNotNull(adResource);
+    Assert.assertEquals(1, adResource.size());
+    Map.Entry<String, ResourceInformation> volume = adResource
+        .entrySet().iterator().next();
+    Assert.assertEquals("yarn.io/csi-volume", volume.getKey());
+    Assert.assertEquals(100L, volume.getValue().getValue().longValue());
+    Assert.assertEquals(2, volume.getValue().getAttributes().size());
+  }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f3a708/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external3.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external3.json
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external3.json
new file mode 100644
index 0000000..74569bd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external3.json
@@ -0,0 +1,26 @@
+{
+  "name": "external-3",
+  "version": "1.0.0",
+  "lifetime": "3600",
+  "components": [
+    {
+      "name": "volume-service",
+      "launch_command": "sleep 3600",
+      "number_of_containers": 1,
+      "resource": {
+        "": 1,
+        "memory": "512",
+        "additional": {
+          "yarn.io/csi-volume": {
+            "value": 100,
+            "unit": "Gi",
+            "attributes" : {
+              "driver" : "hostpath",
+              "mountPath" : "/mnt/data"
+            }
+          }
+        }
+      }
+    }
+  ]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f3a708/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
index fe49158..4ee15e1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
@@ -371,7 +371,7 @@ Resource determines the amount of resources (vcores, memory, network,
etc.) usab
 |profile|Each resource profile has a unique id which is associated with a cluster-level predefined
memory, cpus, etc.|false|string||
 |cpus|Amount of vcores allocated to each container (optional but overrides cpus in profile
if specified).|false|integer (int32)||
 |memory|Amount of memory allocated to each container (optional but overrides memory in profile
if specified). Currently accepts only an integer value and default unit is in MB.|false|string||
-|additional|A map of resource type name to resource type information. Including value (integer),
and unit (string). This will be used to specify resource other than cpu and memory. Please
refer to example below.|false|object||
+|additional|A map of resource type name to resource type information. Including value (integer),
unit (string) and optional attributes (map). This will be used to specify resource other than
cpu and memory. Please refer to example below.|false|object||
 
 
 ### ResourceInformation


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message