hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bha...@apache.org
Subject [02/50] [abbrv] hadoop git commit: YARN-7451. Add missing tests to verify the presence of custom resources of RM apps and scheduler webservice endpoints (snemeth via rkanter)
Date Mon, 09 Jul 2018 20:19:38 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoXmlVerifications.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoXmlVerifications.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoXmlVerifications.java
new file mode 100644
index 0000000..7c5b6db
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoXmlVerifications.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
+
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
+import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
+import org.w3c.dom.Element;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.checkStringMatch;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlBoolean;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlFloat;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlInt;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlLong;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlString;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Contains all value verifications that are needed to verify {@link AppInfo}
+ * XML documents.
+ */
+public final class AppInfoXmlVerifications {
+
+  private AppInfoXmlVerifications() {
+    //utility class
+  }
+
+  /**
+   * Tests whether {@link AppInfo} representation object contains the required
+   * values as per defined in the specified app parameter.
+   * @param info
+   * @param  app  an RMApp instance that contains the required values
+   */
+  public static void verify(Element info, RMApp app) {
+    checkStringMatch("id", app.getApplicationId()
+            .toString(), getXmlString(info, "id"));
+    checkStringMatch("user", app.getUser(),
+            getXmlString(info, "user"));
+    checkStringMatch("name", app.getName(),
+            getXmlString(info, "name"));
+    checkStringMatch("applicationType",
+            app.getApplicationType(), getXmlString(info, "applicationType"));
+    checkStringMatch("queue", app.getQueue(),
+            getXmlString(info, "queue"));
+    assertEquals("priority doesn't match", 0, getXmlInt(info, "priority"));
+    checkStringMatch("state", app.getState().toString(),
+            getXmlString(info, "state"));
+    checkStringMatch("finalStatus", app
+            .getFinalApplicationStatus().toString(),
+            getXmlString(info, "finalStatus"));
+    assertEquals("progress doesn't match", 0, getXmlFloat(info, "progress"),
+        0.0);
+    if ("UNASSIGNED".equals(getXmlString(info, "trackingUI"))) {
+      checkStringMatch("trackingUI", "UNASSIGNED",
+              getXmlString(info, "trackingUI"));
+    }
+    WebServicesTestUtils.checkStringEqual("diagnostics",
+            app.getDiagnostics().toString(), getXmlString(info, "diagnostics"));
+    assertEquals("clusterId doesn't match",
+            ResourceManager.getClusterTimeStamp(),
+            getXmlLong(info, "clusterId"));
+    assertEquals("startedTime doesn't match", app.getStartTime(),
+            getXmlLong(info, "startedTime"));
+    assertEquals("finishedTime doesn't match", app.getFinishTime(),
+            getXmlLong(info, "finishedTime"));
+    assertTrue("elapsed time not greater than 0",
+            getXmlLong(info, "elapsedTime") > 0);
+    checkStringMatch("amHostHttpAddress", app
+                    .getCurrentAppAttempt().getMasterContainer()
+                    .getNodeHttpAddress(),
+            getXmlString(info, "amHostHttpAddress"));
+    assertTrue("amContainerLogs doesn't match",
+        getXmlString(info, "amContainerLogs").startsWith("http://"));
+    assertTrue("amContainerLogs doesn't contain user info",
+        getXmlString(info, "amContainerLogs").endsWith("/" + app.getUser()));
+    assertEquals("allocatedMB doesn't match", 1024,
+            getXmlInt(info, "allocatedMB"));
+    assertEquals("allocatedVCores doesn't match", 1,
+            getXmlInt(info, "allocatedVCores"));
+    assertEquals("queueUsagePerc doesn't match", 50.0f,
+            getXmlFloat(info, "queueUsagePercentage"), 0.01f);
+    assertEquals("clusterUsagePerc doesn't match", 50.0f,
+            getXmlFloat(info, "clusterUsagePercentage"), 0.01f);
+    assertEquals("numContainers doesn't match", 1,
+        getXmlInt(info, "runningContainers"));
+    assertNotNull("preemptedResourceSecondsMap should not be null",
+            info.getElementsByTagName("preemptedResourceSecondsMap"));
+    assertEquals("preemptedResourceMB doesn't match", app
+                    .getRMAppMetrics().getResourcePreempted().getMemorySize(),
+            getXmlInt(info, "preemptedResourceMB"));
+    assertEquals("preemptedResourceVCores doesn't match", app
+                    .getRMAppMetrics().getResourcePreempted().getVirtualCores(),
+            getXmlInt(info, "preemptedResourceVCores"));
+    assertEquals("numNonAMContainerPreempted doesn't match", app
+                    .getRMAppMetrics().getNumNonAMContainersPreempted(),
+            getXmlInt(info, "numNonAMContainerPreempted"));
+    assertEquals("numAMContainerPreempted doesn't match", app
+                    .getRMAppMetrics().getNumAMContainersPreempted(),
+            getXmlInt(info, "numAMContainerPreempted"));
+    assertEquals("Log aggregation Status doesn't match", app
+                    .getLogAggregationStatusForAppReport().toString(),
+            getXmlString(info, "logAggregationStatus"));
+    assertEquals("unmanagedApplication doesn't match", app
+                    .getApplicationSubmissionContext().getUnmanagedAM(),
+            getXmlBoolean(info, "unmanagedApplication"));
+    assertEquals("unmanagedApplication doesn't match",
+            app.getApplicationSubmissionContext().getNodeLabelExpression(),
+            getXmlString(info, "appNodeLabelExpression"));
+    assertEquals("unmanagedApplication doesn't match",
+            app.getAMResourceRequests().get(0).getNodeLabelExpression(),
+            getXmlString(info, "amNodeLabelExpression"));
+    assertEquals("amRPCAddress",
+            AppInfo.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()),
+            getXmlString(info, "amRPCAddress"));
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/BufferedClientResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/BufferedClientResponse.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/BufferedClientResponse.java
new file mode 100644
index 0000000..a8990ca
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/BufferedClientResponse.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
+
+
+import com.sun.jersey.api.client.ClientHandlerException;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.UniformInterfaceException;
+
+import javax.ws.rs.core.MediaType;
+import java.io.IOException;
+
+/**
+ * This class is merely a wrapper for {@link ClientResponse}. Given that the
+ * entity input stream of {@link ClientResponse} can be read only once by
+ * default and for some tests it is convenient to read the input stream many
+ * times, this class hides the details of how to do that and prevents
+ * unnecessary code duplication in tests.
+ */
+public class BufferedClientResponse {
+  private ClientResponse response;
+
+  public BufferedClientResponse(ClientResponse response) {
+    response.bufferEntity();
+    this.response = response;
+  }
+
+  public <T> T getEntity(Class<T> clazz)
+          throws ClientHandlerException, UniformInterfaceException {
+    try {
+      response.getEntityInputStream().reset();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    return response.getEntity(clazz);
+  }
+
+  public MediaType getType() {
+    return response.getType();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/JsonCustomResourceTypeTestcase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/JsonCustomResourceTypeTestcase.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/JsonCustomResourceTypeTestcase.java
new file mode 100644
index 0000000..9d6a111
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/JsonCustomResourceTypeTestcase.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
+
+import com.sun.jersey.api.client.WebResource;
+import org.apache.hadoop.http.JettyUtils;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.core.MediaType;
+
+import java.util.function.Consumer;
+
+import static org.junit.Assert.*;
+
+/**
+ * This class hides the implementation details of how to verify the structure of
+ * JSON responses. Tests should only provide the path of the
+ * {@link WebResource}, the response from the resource and
+ * the verifier Consumer to
+ * {@link JsonCustomResourceTypeTestcase#verify(Consumer)}. An instance of
+ * {@link JSONObject} will be passed to that consumer to be able to
+ * verify the response.
+ */
+public class JsonCustomResourceTypeTestcase {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(JsonCustomResourceTypeTestcase.class);
+
+  private final WebResource path;
+  private final BufferedClientResponse response;
+  private final JSONObject parsedResponse;
+
+  public JsonCustomResourceTypeTestcase(WebResource path,
+                                        BufferedClientResponse response) {
+    this.path = path;
+    this.response = response;
+    this.parsedResponse = response.getEntity(JSONObject.class);
+  }
+
+  public void verify(Consumer<JSONObject> verifier) {
+    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+        response.getType().toString());
+
+    logResponse();
+
+    String responseStr = response.getEntity(String.class);
+    if (responseStr == null || responseStr.isEmpty()) {
+      throw new IllegalStateException("Response is null or empty!");
+    }
+    verifier.accept(parsedResponse);
+  }
+
+  private void logResponse() {
+    String responseStr = response.getEntity(String.class);
+    LOG.info("Raw response from service URL {}: {}", path.toString(),
+        responseStr);
+    LOG.info("Parsed response from service URL {}: {}", path.toString(),
+        parsedResponse);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsJsonVerifications.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsJsonVerifications.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsJsonVerifications.java
new file mode 100644
index 0000000..6e58a89
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsJsonVerifications.java
@@ -0,0 +1,252 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+import java.util.List;
+import java.util.Map;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * Performs value verifications on
+ * {@link org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceRequestInfo}
+ * objects against the values of {@link ResourceRequest}. With the help of the
+ * {@link Builder}, users can also make verifications of the custom resource
+ * types and its values.
+ */
+public class ResourceRequestsJsonVerifications {
+  private final ResourceRequest resourceRequest;
+  private final JSONObject requestInfo;
+  private final Map<String, Long> customResourceTypes;
+  private final List<String> expectedCustomResourceTypes;
+
+  ResourceRequestsJsonVerifications(Builder builder) {
+    this.resourceRequest = builder.resourceRequest;
+    this.requestInfo = builder.requestInfo;
+    this.customResourceTypes = builder.customResourceTypes;
+    this.expectedCustomResourceTypes = builder.expectedCustomResourceTypes;
+  }
+
+  public static void verify(JSONObject requestInfo, ResourceRequest rr)
+      throws JSONException {
+    createDefaultBuilder(requestInfo, rr).build().verify();
+  }
+
+  public static void verifyWithCustomResourceTypes(JSONObject requestInfo,
+      ResourceRequest resourceRequest, List<String> expectedResourceTypes)
+      throws JSONException {
+
+    createDefaultBuilder(requestInfo, resourceRequest)
+        .withExpectedCustomResourceTypes(expectedResourceTypes)
+        .withCustomResourceTypes(
+            extractActualCustomResourceTypes(requestInfo, expectedResourceTypes))
+        .build().verify();
+  }
+
+  private static Builder createDefaultBuilder(JSONObject requestInfo,
+      ResourceRequest resourceRequest) {
+    return new ResourceRequestsJsonVerifications.Builder()
+            .withRequest(resourceRequest)
+            .withRequestInfoJson(requestInfo);
+  }
+
+  private static Map<String, Long> extractActualCustomResourceTypes(
+      JSONObject requestInfo, List<String> expectedResourceTypes)
+      throws JSONException {
+    JSONObject capability = requestInfo.getJSONObject("capability");
+    Map<String, Long> resourceAndValue =
+        extractCustomResorceTypeValues(capability, expectedResourceTypes);
+    Map.Entry<String, Long> resourceEntry =
+        resourceAndValue.entrySet().iterator().next();
+
+    assertTrue(
+        "Found resource type: " + resourceEntry.getKey()
+            + " is not in expected resource types: " + expectedResourceTypes,
+        expectedResourceTypes.contains(resourceEntry.getKey()));
+
+    return resourceAndValue;
+  }
+
+  private static Map<String, Long> extractCustomResorceTypeValues(
+      JSONObject capability, List<String> expectedResourceTypes)
+      throws JSONException {
+    assertTrue(
+        "resourceCategory does not have resourceInformations: " + capability,
+        capability.has("resourceInformations"));
+
+    JSONObject resourceInformations =
+        capability.getJSONObject("resourceInformations");
+    assertTrue(
+        "resourceInformations does not have resourceInformation object: "
+            + resourceInformations,
+        resourceInformations.has("resourceInformation"));
+    JSONArray customResources =
+        resourceInformations.getJSONArray("resourceInformation");
+
+    // customResources will include vcores / memory as well
+    assertEquals(
+        "Different number of custom resource types found than expected",
+        expectedResourceTypes.size(), customResources.length() - 2);
+
+    Map<String, Long> resourceValues = Maps.newHashMap();
+    for (int i = 0; i < customResources.length(); i++) {
+      JSONObject customResource = customResources.getJSONObject(i);
+      assertTrue("Resource type does not have name field: " + customResource,
+          customResource.has("name"));
+      assertTrue("Resource type does not have name resourceType field: "
+          + customResource, customResource.has("resourceType"));
+      assertTrue(
+          "Resource type does not have name units field: " + customResource,
+          customResource.has("units"));
+      assertTrue(
+          "Resource type does not have name value field: " + customResource,
+          customResource.has("value"));
+
+      String name = customResource.getString("name");
+      String unit = customResource.getString("units");
+      String resourceType = customResource.getString("resourceType");
+      Long value = customResource.getLong("value");
+
+      if (ResourceInformation.MEMORY_URI.equals(name)
+          || ResourceInformation.VCORES_URI.equals(name)) {
+        continue;
+      }
+
+      assertTrue("Custom resource type " + name + " not found",
+          expectedResourceTypes.contains(name));
+      assertEquals("k", unit);
+      assertEquals(ResourceTypes.COUNTABLE,
+          ResourceTypes.valueOf(resourceType));
+      assertNotNull("Custom resource value " + value + " is null!", value);
+      resourceValues.put(name, value);
+    }
+
+    return resourceValues;
+  }
+
+  private void verify() throws JSONException {
+    assertEquals("nodeLabelExpression doesn't match",
+        resourceRequest.getNodeLabelExpression(),
+            requestInfo.getString("nodeLabelExpression"));
+    assertEquals("numContainers doesn't match",
+            resourceRequest.getNumContainers(),
+            requestInfo.getInt("numContainers"));
+    assertEquals("relaxLocality doesn't match",
+            resourceRequest.getRelaxLocality(),
+            requestInfo.getBoolean("relaxLocality"));
+    assertEquals("priority does not match",
+            resourceRequest.getPriority().getPriority(),
+            requestInfo.getInt("priority"));
+    assertEquals("resourceName does not match",
+            resourceRequest.getResourceName(),
+            requestInfo.getString("resourceName"));
+    assertEquals("memory does not match",
+        resourceRequest.getCapability().getMemorySize(),
+            requestInfo.getJSONObject("capability").getLong("memory"));
+    assertEquals("vCores does not match",
+        resourceRequest.getCapability().getVirtualCores(),
+            requestInfo.getJSONObject("capability").getLong("vCores"));
+
+    verifyAtLeastOneCustomResourceIsSerialized();
+
+    JSONObject executionTypeRequest =
+            requestInfo.getJSONObject("executionTypeRequest");
+    assertEquals("executionType does not match",
+        resourceRequest.getExecutionTypeRequest().getExecutionType().name(),
+            executionTypeRequest.getString("executionType"));
+    assertEquals("enforceExecutionType does not match",
+            resourceRequest.getExecutionTypeRequest().getEnforceExecutionType(),
+            executionTypeRequest.getBoolean("enforceExecutionType"));
+  }
+
+  /**
+   * JSON serialization produces "invalid JSON" by default as maps are
+   * serialized like this:
+   * "customResources":{"entry":{"key":"customResource-1","value":"0"}}
+   * If the map has multiple keys then multiple entries will be serialized.
+   * Our json parser in tests cannot handle duplicates therefore only one
+   * custom resource will be in the parsed json. See:
+   * https://issues.apache.org/jira/browse/YARN-7505
+   */
+  private void verifyAtLeastOneCustomResourceIsSerialized() {
+    boolean resourceFound = false;
+    for (String expectedCustomResourceType : expectedCustomResourceTypes) {
+      if (customResourceTypes.containsKey(expectedCustomResourceType)) {
+        resourceFound = true;
+        Long resourceValue =
+            customResourceTypes.get(expectedCustomResourceType);
+        assertNotNull("Resource value should not be null!", resourceValue);
+      }
+    }
+    assertTrue("No custom resource type can be found in the response!",
+        resourceFound);
+  }
+
+  /**
+   * Builder class for {@link ResourceRequestsJsonVerifications}.
+   */
+  public static final class Builder {
+    private List<String> expectedCustomResourceTypes = Lists.newArrayList();
+    private Map<String, Long> customResourceTypes;
+    private ResourceRequest resourceRequest;
+    private JSONObject requestInfo;
+
+    Builder() {
+    }
+
+    public static Builder create() {
+      return new Builder();
+    }
+
+    Builder withExpectedCustomResourceTypes(
+            List<String> expectedCustomResourceTypes) {
+      this.expectedCustomResourceTypes = expectedCustomResourceTypes;
+      return this;
+    }
+
+    Builder withCustomResourceTypes(
+            Map<String, Long> customResourceTypes) {
+      this.customResourceTypes = customResourceTypes;
+      return this;
+    }
+
+    Builder withRequest(ResourceRequest resourceRequest) {
+      this.resourceRequest = resourceRequest;
+      return this;
+    }
+
+    Builder withRequestInfoJson(JSONObject requestInfo) {
+      this.requestInfo = requestInfo;
+      return this;
+    }
+
+    public ResourceRequestsJsonVerifications build() {
+      return new ResourceRequestsJsonVerifications(this);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsXmlVerifications.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsXmlVerifications.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsXmlVerifications.java
new file mode 100644
index 0000000..af9b0f3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsXmlVerifications.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.XmlCustomResourceTypeTestCase.toXml;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlBoolean;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlInt;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlLong;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlString;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * Performs value verifications on
+ * {@link org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceRequestInfo}
+ * objects against the values of {@link ResourceRequest}. With the help of the
+ * {@link Builder}, users can also make verifications of the custom resource
+ * types and its values.
+ */
+public class ResourceRequestsXmlVerifications {
+  private final ResourceRequest resourceRequest;
+  private final Element requestInfo;
+  private final Map<String, Long> customResourceTypes;
+  private final List<String> expectedCustomResourceTypes;
+
+  ResourceRequestsXmlVerifications(Builder builder) {
+    this.resourceRequest = builder.resourceRequest;
+    this.requestInfo = builder.requestInfo;
+    this.customResourceTypes = builder.customResourceTypes;
+    this.expectedCustomResourceTypes = builder.expectedCustomResourceTypes;
+  }
+
+  public static void verifyWithCustomResourceTypes(Element requestInfo,
+      ResourceRequest resourceRequest, List<String> expectedResourceTypes) {
+
+    createDefaultBuilder(requestInfo, resourceRequest)
+        .withExpectedCustomResourceTypes(expectedResourceTypes)
+        .withCustomResourceTypes(extractActualCustomResourceType(requestInfo,
+            expectedResourceTypes))
+        .build().verify();
+  }
+
+  private static Builder createDefaultBuilder(Element requestInfo,
+      ResourceRequest resourceRequest) {
+    return new ResourceRequestsXmlVerifications.Builder()
+        .withRequest(resourceRequest).withRequestInfo(requestInfo);
+  }
+
+  private static Map<String, Long> extractActualCustomResourceType(
+      Element requestInfo, List<String> expectedResourceTypes) {
+    Element capability =
+        (Element) requestInfo.getElementsByTagName("capability").item(0);
+
+    return extractCustomResorceTypes(capability,
+        Sets.newHashSet(expectedResourceTypes));
+  }
+
+  private static Map<String, Long> extractCustomResorceTypes(Element capability,
+      Set<String> expectedResourceTypes) {
+    assertEquals(
+        toXml(capability) + " should have only one resourceInformations child!",
+        1, capability.getElementsByTagName("resourceInformations").getLength());
+    Element resourceInformations = (Element) capability
+        .getElementsByTagName("resourceInformations").item(0);
+
+    NodeList customResources =
+        resourceInformations.getElementsByTagName("resourceInformation");
+
+    // customResources will include vcores / memory as well
+    assertEquals(
+        "Different number of custom resource types found than expected",
+        expectedResourceTypes.size(), customResources.getLength() - 2);
+
+    Map<String, Long> resourceTypesAndValues = Maps.newHashMap();
+    for (int i = 0; i < customResources.getLength(); i++) {
+      Element customResource = (Element) customResources.item(i);
+      String name = getXmlString(customResource, "name");
+      String unit = getXmlString(customResource, "units");
+      String resourceType = getXmlString(customResource, "resourceType");
+      Long value = getXmlLong(customResource, "value");
+
+      if (ResourceInformation.MEMORY_URI.equals(name)
+          || ResourceInformation.VCORES_URI.equals(name)) {
+        continue;
+      }
+
+      assertTrue("Custom resource type " + name + " not found",
+          expectedResourceTypes.contains(name));
+      assertEquals("k", unit);
+      assertEquals(ResourceTypes.COUNTABLE,
+          ResourceTypes.valueOf(resourceType));
+      assertNotNull("Resource value should not be null for resource type "
+          + resourceType + ", listing xml contents: " + toXml(customResource),
+          value);
+      resourceTypesAndValues.put(name, value);
+    }
+
+    return resourceTypesAndValues;
+  }
+
+  private void verify() {
+    assertEquals("nodeLabelExpression doesn't match",
+        resourceRequest.getNodeLabelExpression(),
+        getXmlString(requestInfo, "nodeLabelExpression"));
+    assertEquals("numContainers doesn't match",
+        resourceRequest.getNumContainers(),
+        getXmlInt(requestInfo, "numContainers"));
+    assertEquals("relaxLocality doesn't match",
+        resourceRequest.getRelaxLocality(),
+        getXmlBoolean(requestInfo, "relaxLocality"));
+    assertEquals("priority does not match",
+        resourceRequest.getPriority().getPriority(),
+        getXmlInt(requestInfo, "priority"));
+    assertEquals("resourceName does not match",
+        resourceRequest.getResourceName(),
+        getXmlString(requestInfo, "resourceName"));
+    Element capability = (Element) requestInfo
+            .getElementsByTagName("capability").item(0);
+    assertEquals("memory does not match",
+        resourceRequest.getCapability().getMemorySize(),
+        getXmlLong(capability, "memory"));
+    assertEquals("vCores does not match",
+        resourceRequest.getCapability().getVirtualCores(),
+        getXmlLong(capability, "vCores"));
+
+    for (String expectedCustomResourceType : expectedCustomResourceTypes) {
+      assertTrue(
+          "Custom resource type " + expectedCustomResourceType
+              + " cannot be found!",
+          customResourceTypes.containsKey(expectedCustomResourceType));
+
+      Long resourceValue = customResourceTypes.get(expectedCustomResourceType);
+      assertNotNull("Resource value should not be null!", resourceValue);
+    }
+
+    Element executionTypeRequest = (Element) requestInfo
+        .getElementsByTagName("executionTypeRequest").item(0);
+    assertEquals("executionType does not match",
+        resourceRequest.getExecutionTypeRequest().getExecutionType().name(),
+        getXmlString(executionTypeRequest, "executionType"));
+    assertEquals("enforceExecutionType does not match",
+        resourceRequest.getExecutionTypeRequest().getEnforceExecutionType(),
+        getXmlBoolean(executionTypeRequest, "enforceExecutionType"));
+  }
+
+  /**
+   * Builder class for {@link ResourceRequestsXmlVerifications}.
+   */
+  public static final class Builder {
+    private List<String> expectedCustomResourceTypes = Lists.newArrayList();
+    private Map<String, Long> customResourceTypes;
+    private ResourceRequest resourceRequest;
+    private Element requestInfo;
+
+    Builder() {
+    }
+
+    public static Builder create() {
+      return new Builder();
+    }
+
+    Builder withExpectedCustomResourceTypes(
+        List<String> expectedCustomResourceTypes) {
+      this.expectedCustomResourceTypes = expectedCustomResourceTypes;
+      return this;
+    }
+
+    Builder withCustomResourceTypes(Map<String, Long> customResourceTypes) {
+      this.customResourceTypes = customResourceTypes;
+      return this;
+    }
+
+    Builder withRequest(ResourceRequest resourceRequest) {
+      this.resourceRequest = resourceRequest;
+      return this;
+    }
+
+    Builder withRequestInfo(Element requestInfo) {
+      this.requestInfo = requestInfo;
+      return this;
+    }
+
+    public ResourceRequestsXmlVerifications build() {
+      return new ResourceRequestsXmlVerifications(this);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/XmlCustomResourceTypeTestCase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/XmlCustomResourceTypeTestCase.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/XmlCustomResourceTypeTestCase.java
new file mode 100644
index 0000000..29260aa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/XmlCustomResourceTypeTestCase.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
+
+import com.sun.jersey.api.client.WebResource;
+import org.apache.hadoop.http.JettyUtils;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import org.xml.sax.InputSource;
+
+import javax.ws.rs.core.MediaType;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.transform.*;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.util.function.Consumer;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * This class hides the implementation details of how to verify the structure of
+ * XML responses. Tests should only provide the path of the
+ * {@link WebResource}, the response from the resource and
+ * the verifier Consumer to
+ * {@link XmlCustomResourceTypeTestCase#verify(Consumer)}. An instance of
+ * {@link JSONObject} will be passed to that consumer to be able to
+ * verify the response.
+ */
+public class XmlCustomResourceTypeTestCase {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(XmlCustomResourceTypeTestCase.class);
+
+  private WebResource path;
+  private BufferedClientResponse response;
+  private Document parsedResponse;
+
+  public XmlCustomResourceTypeTestCase(WebResource path,
+                                       BufferedClientResponse response) {
+    this.path = path;
+    this.response = response;
+  }
+
+  public void verify(Consumer<Document> verifier) {
+    assertEquals(MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8,
+        response.getType().toString());
+
+    parsedResponse = parseXml(response);
+    logResponse(parsedResponse);
+    verifier.accept(parsedResponse);
+  }
+
+  private Document parseXml(BufferedClientResponse response) {
+    try {
+      String xml = response.getEntity(String.class);
+      DocumentBuilder db =
+          DocumentBuilderFactory.newInstance().newDocumentBuilder();
+      InputSource is = new InputSource();
+      is.setCharacterStream(new StringReader(xml));
+
+      return db.parse(is);
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private void logResponse(Document doc) {
+    String responseStr = response.getEntity(String.class);
+    LOG.info("Raw response from service URL {}: {}", path.toString(),
+        responseStr);
+    LOG.info("Parsed response from service URL {}: {}", path.toString(),
+        toXml(doc));
+  }
+
+  public static String toXml(Node node) {
+    StringWriter writer;
+    try {
+      TransformerFactory tf = TransformerFactory.newInstance();
+      Transformer transformer = tf.newTransformer();
+      transformer.setOutputProperty(OutputKeys.INDENT, "yes");
+      transformer.setOutputProperty(
+          "{http://xml.apache.org/xslt}indent" + "-amount", "2");
+      writer = new StringWriter();
+      transformer.transform(new DOMSource(node), new StreamResult(writer));
+    } catch (TransformerException e) {
+      throw new RuntimeException(e);
+    }
+
+    return writer.getBuffer().toString();
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message