hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jia...@apache.org
Subject [48/50] [abbrv] hadoop git commit: YARN-6716. Native services support for specifying component start order. Contributed by Billie Rinaldi
Date Tue, 25 Jul 2017 18:03:30 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14b8371b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/ReportingLoop.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/ReportingLoop.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/ReportingLoop.java
deleted file mode 100644
index 096838d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/ReportingLoop.java
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.servicemonitor;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-
-/**
- * This is the monitor service
- */
-public final class ReportingLoop implements Runnable, ProbeReportHandler, MonitorKeys, Closeable
{
-  protected static final Logger log = LoggerFactory.getLogger(ReportingLoop.class);
-  private final ProbeWorker worker;
-  private final Thread workerThread;
-  private final int reportInterval;
-  private final int probeTimeout;
-  private final int bootstrapTimeout;
-  private ProbeReportHandler reporter;
-  private final String name;
-  private volatile boolean mustExit;
-
-  public ReportingLoop(String name,
-                       ProbeReportHandler reporter,
-                       List<Probe> probes,
-                       List<Probe> dependencyProbes,
-                       int probeInterval,
-                       int reportInterval,
-                       int probeTimeout,
-                       int bootstrapTimeout) throws IOException {
-    this(name,
-         reporter,
-         new ProbeWorker(probes, dependencyProbes, probeInterval, bootstrapTimeout),
-         reportInterval,
-         probeTimeout);
-  }
-
-  /**
-   * Create a new reporting loop -and bond the worker's ProbeReportHandler
-   * to us
-   * @param name
-   * @param reporter
-   * @param worker
-   * @param reportInterval
-   * @param probeTimeout
-   */
-  public ReportingLoop(String name,
-                       ProbeReportHandler reporter,
-                       ProbeWorker worker,
-                       int reportInterval,
-                       int probeTimeout) throws IOException {
-    this.name = name;
-    this.reporter = reporter;
-    this.reportInterval = reportInterval;
-    this.probeTimeout = probeTimeout;
-    this.worker = worker;
-    this.bootstrapTimeout = worker.getBootstrapTimeout();
-    worker.setReportHandler(this);
-    workerThread = new Thread(worker, "probe thread - " + name);
-    worker.init();
-  }
-  
-  public int getBootstrapTimeout() {
-    return bootstrapTimeout;
-  }
-
-  public ReportingLoop withReporter(ProbeReportHandler reporter) {
-    assert this.reporter == null : "attempting to reassign reporter ";
-    assert reporter != null : "new reporter is null";
-    this.reporter = reporter;
-    return this;
-  }
-
-  /**
-   * Start the monitoring.
-   *
-   * @return false if the monitoring did not start and that the worker threads
-   *         should be run up.
-   */
-  public boolean startReporting() {
-    String description = "Service Monitor for " + name + ", probe-interval= "
-                         + MonitorUtils.millisToHumanTime(worker.interval)
-                         + ", report-interval=" + MonitorUtils.millisToHumanTime(reportInterval)
-                         + ", probe-timeout=" + timeoutToStr(probeTimeout)
-                         + ", bootstrap-timeout=" + timeoutToStr(bootstrapTimeout);
-    log.info("Starting reporting"
-             + " to " + reporter
-             + description);
-    return reporter.commence(name, description);
-  }
-
-  private String timeoutToStr(int timeout) {
-    return timeout >= 0 ? MonitorUtils.millisToHumanTime(timeout) : "not set";
-  }
-
-  private void startWorker() {
-    log.info("Starting reporting worker thread ");
-    workerThread.setDaemon(true);
-    workerThread.start();
-  }
-
-
-  /**
-   * This exits the process cleanly
-   */
-  @Override
-  public void close() {
-    log.info("Stopping reporting");
-    mustExit = true;
-    if (worker != null) {
-      worker.setMustExit();
-      workerThread.interrupt();
-    }
-    if (reporter != null) {
-      reporter.unregister();
-    }
-  }
-
-  @Override
-  public void probeFailure(ProbeFailedException exception) {
-    reporter.probeFailure(exception);
-  }
-
-  @Override
-  public void probeProcessStateChange(ProbePhase probePhase) {
-    reporter.probeProcessStateChange(probePhase);
-  }
-
-  @Override
-  public void probeBooted(ProbeStatus status) {
-    reporter.probeBooted(status);
-  }
-
-  private long now() {
-    return System.currentTimeMillis();
-  }
-
-  @Override
-  public void probeResult(ProbePhase phase, ProbeStatus status) {
-    reporter.probeResult(phase, status);
-  }
-
-  @Override
-  public boolean commence(String n, String description) {
-    return true;
-  }
-
-  @Override
-  public void unregister() {
-  }
-
-  @Override
-  public void heartbeat(ProbeStatus status) {
-  }
-
-  @Override
-  public void probeTimedOut(ProbePhase currentPhase, Probe probe, ProbeStatus lastStatus,
-      long currentTime) {
-  }
-
-  @Override
-  public void liveProbeCycleCompleted() {
-    //delegate to the reporter
-    reporter.liveProbeCycleCompleted();
-  }
-
-  /**
-   * The reporting loop
-   */
-  void reportingLoop() {
-
-    while (!mustExit) {
-      try {
-        ProbeStatus workerStatus = worker.getLastStatus();
-        long now = now();
-        long lastStatusIssued = workerStatus.getTimestamp();
-        long timeSinceLastStatusIssued = now - lastStatusIssued;
-        //two actions can occur here: a heartbeat is issued or a timeout reported. 
-        //this flag decides which
-        boolean heartbeat;
-
-        //based on phase, decide whether to heartbeat or timeout
-        ProbePhase probePhase = worker.getProbePhase();
-        switch (probePhase) {
-          case DEPENDENCY_CHECKING:
-            //no timeouts in dependency phase
-            heartbeat = true;
-            break;
-
-          case BOOTSTRAPPING:
-            //the timeout here is fairly straightforward: heartbeats are
-            //raised while the worker hasn't timed out
-            heartbeat = bootstrapTimeout < 0 || timeSinceLastStatusIssued < bootstrapTimeout;
-
-            break;
-
-          case LIVE:
-            //use the probe timeout interval between the current time
-            //and the time the last status event was received.
-            heartbeat = timeSinceLastStatusIssued < probeTimeout;
-            break;
-
-          case INIT:
-          case TERMINATING:
-          default:
-            //send a heartbeat, because this isn't the time to be failing
-            heartbeat = true;
-        }
-        if (heartbeat) {
-          //a heartbeat is sent to the reporter
-          reporter.heartbeat(workerStatus);
-        } else {
-          //no response from the worker -it is hung.
-          reporter.probeTimedOut(probePhase,
-                                 worker.getCurrentProbe(),
-                                 workerStatus,
-                                 now
-                                );
-        }
-
-        //now sleep
-        Thread.sleep(reportInterval);
-
-      } catch (InterruptedException e) {
-        //interrupted -always exit the loop.
-        break;
-      }
-    }
-    //this point is reached if and only if a clean exit was requested or something failed.
-  }
-
-  /**
-   * This can be run in a separate thread, or it can be run directly from the caller.
-   * Test runs do the latter, HAM runs multiple reporting threads.
-   */
-  @Override
-  public void run() {
-    try {
-      startWorker();
-      reportingLoop();
-    } catch (RuntimeException e) {
-      log.warn("Failure in the reporting loop: " + e, e);
-      //rethrow so that inline code can pick it up (e.g. test runs)
-      throw e;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/14b8371b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
index 676db82..3033537 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
@@ -62,6 +62,10 @@ public interface RestApiErrorMessages {
       "Invalid no of containers specified";
   String ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID =
       ERROR_CONTAINERS_COUNT_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+  String ERROR_DEPENDENCY_INVALID = "Dependency %s for component %s is " +
+      "invalid, does not exist as a component";
+  String ERROR_DEPENDENCY_CYCLE = "Invalid dependencies, a cycle may " +
+      "exist: %s";
 
   String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED =
       "Cannot specify" + " cpus/memory along with profile";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/14b8371b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
index 80a31c0..e977727 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
@@ -32,14 +32,18 @@ import org.apache.slider.common.tools.SliderUtils;
 import org.apache.slider.core.persist.JsonSerDeser;
 import org.apache.slider.providers.AbstractClientProvider;
 import org.apache.slider.providers.SliderProviderFactory;
+import org.apache.slider.server.servicemonitor.MonitorUtils;
 import org.codehaus.jackson.map.PropertyNamingStrategy;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 public class ServiceApiUtil {
@@ -176,9 +180,22 @@ public class ServiceApiUtil {
       if (comp.getLaunchCommand() == null) {
         comp.setLaunchCommand(globalLaunchCommand);
       }
+      // validate dependency existence
+      if (comp.getDependencies() != null) {
+        for (String dependency : comp.getDependencies()) {
+          if (!componentNames.contains(dependency)) {
+            throw new IllegalArgumentException(String.format(
+                RestApiErrorMessages.ERROR_DEPENDENCY_INVALID, dependency,
+                comp.getName()));
+          }
+        }
+      }
       validateComponent(comp, fs.getFileSystem());
     }
 
+    // validate dependency tree
+    sortByDependencies(application.getComponents());
+
     // Application lifetime if not specified, is set to unlimited lifetime
     if (application.getLifetime() == null) {
       application.setLifetime(RestApiConstants.DEFAULT_UNLIMITED_LIFETIME);
@@ -207,6 +224,8 @@ public class ServiceApiUtil {
     }
     compClientProvider.validateConfigFiles(comp.getConfiguration()
         .getFiles(), fs);
+
+    MonitorUtils.getProbe(comp.getReadinessCheck());
   }
 
   @VisibleForTesting
@@ -301,6 +320,67 @@ public class ServiceApiUtil {
     return comp;
   }
 
+  public static Collection<Component> sortByDependencies(List<Component>
+      components) {
+    Map<String, Component> sortedComponents =
+        sortByDependencies(components, null);
+    return sortedComponents.values();
+  }
+
+  /**
+   * Each internal call of sortByDependencies will identify all of the
+   * components with the same dependency depth (the lowest depth that has not
+   * been processed yet) and add them to the sortedComponents list, preserving
+   * their original ordering in the components list.
+   *
+   * So the first time it is called, all components with no dependencies
+   * (depth 0) will be identified. The next time it is called, all components
+   * that have dependencies only on the the depth 0 components will be
+   * identified (depth 1). This will be repeated until all components have
+   * been added to the sortedComponents list. If no new components are
+   * identified but the sortedComponents list is not complete, an error is
+   * thrown.
+   */
+  private static Map<String, Component> sortByDependencies(List<Component>
+      components, Map<String, Component> sortedComponents) {
+    if (sortedComponents == null) {
+      sortedComponents = new LinkedHashMap<>();
+    }
+
+    Map<String, Component> componentsToAdd = new LinkedHashMap<>();
+    List<Component> componentsSkipped = new ArrayList<>();
+    for (Component component : components) {
+      String name = component.getName();
+      if (sortedComponents.containsKey(name)) {
+        continue;
+      }
+      boolean dependenciesAlreadySorted = true;
+      if (!SliderUtils.isEmpty(component.getDependencies())) {
+        for (String dependency : component.getDependencies()) {
+          if (!sortedComponents.containsKey(dependency)) {
+            dependenciesAlreadySorted = false;
+            break;
+          }
+        }
+      }
+      if (dependenciesAlreadySorted) {
+        componentsToAdd.put(name, component);
+      } else {
+        componentsSkipped.add(component);
+      }
+    }
+
+    if (componentsToAdd.size() == 0) {
+      throw new IllegalArgumentException(String.format(RestApiErrorMessages
+          .ERROR_DEPENDENCY_CYCLE, componentsSkipped));
+    }
+    sortedComponents.putAll(componentsToAdd);
+    if (sortedComponents.size() == components.size()) {
+      return sortedComponents;
+    }
+    return sortByDependencies(components, sortedComponents);
+  }
+
   public static String $(String s) {
     return "${" + s +"}";
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/14b8371b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDependencies.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDependencies.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDependencies.java
new file mode 100644
index 0000000..2967309
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDependencies.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.model.appstate;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.slider.api.types.ApplicationLivenessInformation;
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
+import org.apache.slider.server.appmaster.model.mock.MockRoles;
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
+import org.apache.slider.server.appmaster.state.ContainerAssignment;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.apache.slider.server.servicemonitor.ProbeStatus;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Test for postponing container requests until dependencies are ready.
+ */
+public class TestMockAppStateDependencies extends BaseMockAppStateTest
+    implements MockRoles {
+
+  private org.apache.slider.server.servicemonitor.Probe successProbe =
+      new org.apache.slider.server.servicemonitor.Probe("success", null) {
+        @Override
+        public ProbeStatus ping(RoleInstance roleInstance) {
+          ProbeStatus status = new ProbeStatus();
+          status.succeed(this);
+          return status;
+        }
+      };
+
+  private org.apache.slider.server.servicemonitor.Probe failureProbe =
+      new org.apache.slider.server.servicemonitor.Probe("failure", null) {
+        @Override
+        public ProbeStatus ping(RoleInstance roleInstance) {
+          ProbeStatus status = new ProbeStatus();
+          status.fail(this, new Exception());
+          return status;
+        }
+      };
+
+  @Override
+  public String getTestName() {
+    return "TestMockAppStateDependencies";
+  }
+
+  @Test
+  public void testDependencies() throws Throwable {
+    RoleStatus role0Status = getRole0Status();
+    RoleStatus role1Status = getRole1Status();
+
+    // set desired instances for role0 to 1
+    role0Status.setDesired(1);
+    // set probe for role0 to use a ping that will always succeed
+    role0Status.getProviderRole().probe = successProbe;
+
+    // set desired instances for role1 to 1
+    role1Status.setDesired(1);
+    // set role0 as a dependency of role1
+    role1Status.getProviderRole().component.setDependencies(Collections
+        .singletonList(ROLE0));
+
+    // role0 has no dependencies, so its dependencies are ready
+    assertTrue(appState.areDependenciesReady(role0Status));
+    // role1 dependency (role0) is not ready yet
+    assertFalse(appState.areDependenciesReady(role1Status));
+    // start the single requested instance for role0
+    review(ROLE0, 2);
+
+    // role0 is still not ready because a ping has not been issued
+    assertFalse(appState.areDependenciesReady(role1Status));
+    // issue pings
+    appState.monitorComponentInstances();
+    // now role0 is ready
+    assertTrue(appState.areDependenciesReady(role1Status));
+    // increase the desired containers for role0
+    role0Status.setDesired(2);
+    // role0 is no longer ready
+    assertFalse(appState.areDependenciesReady(role1Status));
+    // start a second instance for role0
+    review(ROLE0, 2);
+
+    // role0 is not ready because ping has not been issued for the new instance
+    assertFalse(appState.areDependenciesReady(role1Status));
+    // issue pings
+    appState.monitorComponentInstances();
+    // role0 is ready
+    assertTrue(appState.areDependenciesReady(role1Status));
+
+    // set probe for role0 to use a ping that will always fail
+    role0Status.getProviderRole().probe = failureProbe;
+    // issue pings
+    appState.monitorComponentInstances();
+    // role0 is not ready (failure probe works)
+    assertFalse(appState.areDependenciesReady(role1Status));
+    // set probe for role0 to use a ping that will always succeed
+    role0Status.getProviderRole().probe = successProbe;
+    // issue pings
+    appState.monitorComponentInstances();
+    // role0 is ready
+    assertTrue(appState.areDependenciesReady(role1Status));
+
+    // now role1 instances can be started
+    review(ROLE1, 1);
+  }
+
+  public void review(String expectedRole, int outstanding) throws Exception {
+    List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
+
+    // expect one request in the list
+    assertEquals(1, ops.size());
+    // and in a liveness check, expected outstanding
+    ApplicationLivenessInformation liveness =
+        appState.getApplicationLivenessInformation();
+    assertEquals(outstanding, liveness.requestsOutstanding);
+    assertFalse(liveness.allRequestsSatisfied);
+
+    // record container allocated and verify it has the expected role
+    List<Container> allocations = engine.execute(ops);
+    List<ContainerAssignment> assignments = new ArrayList<>();
+    List<AbstractRMOperation> releases = new ArrayList<>();
+    appState.onContainersAllocated(allocations, assignments, releases);
+    assertEquals(1, assignments.size());
+    ContainerAssignment assigned = assignments.get(0);
+    Container target = assigned.container;
+    RoleInstance ri = roleInstance(assigned);
+    assertEquals(expectedRole, ri.role);
+
+    // one fewer request outstanding
+    liveness = appState.getApplicationLivenessInformation();
+    assertEquals(outstanding - 1, liveness.requestsOutstanding);
+
+    // record container start submitted
+    appState.containerStartSubmitted(target, ri);
+
+    // additional review results in no additional requests
+    ops = appState.reviewRequestAndReleaseNodes();
+    assertTrue(ops.isEmpty());
+
+    // record container start
+    appState.innerOnNodeManagerContainerStarted(target.getId());
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/14b8371b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
index 703d65f..edc1866 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
@@ -103,8 +103,6 @@ public class TestMockAppStateUniqueNames extends BaseMockAppStateTest
       assertEquals(i, instance.componentId);
       assertEquals(group, instance.role);
       assertEquals(group, instance.providerRole.name);
-      assertEquals(group, instance.providerRole.group);
-      // TODO remove group from provider role if it continues to be unused
       i++;
     }
   }
@@ -124,7 +122,6 @@ public class TestMockAppStateUniqueNames extends BaseMockAppStateTest
     assertEquals(0, roleStatus.getDesired());
     assertEquals(1024L, roleStatus.getResourceRequirements().getMemorySize());
     assertEquals(2, roleStatus.getResourceRequirements().getVirtualCores());
-    assertEquals("group1", roleStatus.getGroup());
 
     // now flex back up
     appState.updateComponents(Collections.singletonMap("group1", 3L));
@@ -147,7 +144,6 @@ public class TestMockAppStateUniqueNames extends BaseMockAppStateTest
     RoleStatus group1 = appState.lookupRoleStatus("group1");
     assertEquals(3, group1.getDesired());
     assertEquals(1024L, group1.getResourceRequirements().getMemorySize());
-    assertEquals("group1", group1.getGroup());
   }
 
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/14b8371b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java
index 7d8f5a7..555db75 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java
@@ -343,7 +343,7 @@ public class TestRoleHistoryOutstandingRequestTracker extends
   public void testBuildResourceRequirements() throws Throwable {
     // Store original values
     Application application = appState.getClusterStatus();
-    Component role0 = application.getComponent(getRole0Status().getGroup());
+    Component role0 = application.getComponent(getRole0Status().getName());
     String origMem = role0.getResource().getMemory();
     Integer origVcores = role0.getResource().getCpus();
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/14b8371b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/servicemonitor/TestPortProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/servicemonitor/TestPortProbe.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/servicemonitor/TestPortProbe.java
index 458d1bc..dacfb0a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/servicemonitor/TestPortProbe.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/servicemonitor/TestPortProbe.java
@@ -17,20 +17,25 @@
 
 package org.apache.slider.server.servicemonitor;
 
-import org.apache.hadoop.conf.Configuration;
+import org.apache.slider.server.appmaster.model.mock.MockFactory;
+import org.apache.slider.server.appmaster.state.RoleInstance;
 import org.junit.Assert;
 import org.junit.Test;
 
 public class TestPortProbe extends Assert {
+  private final MockFactory factory = MockFactory.INSTANCE;
+
   /**
    * Assert that a port probe failed if the port is closed
    * @throws Throwable
    */
   @Test
   public void testPortProbeFailsClosedPort() throws Throwable {
-    PortProbe probe = new PortProbe("127.0.0.1", 65500, 100, "", new Configuration());
+    PortProbe probe = new PortProbe(65500, 100);
     probe.init();
-    ProbeStatus status = probe.ping(true);
+    RoleInstance roleInstance = new RoleInstance(factory.newContainer());
+    roleInstance.ip = "127.0.0.1";
+    ProbeStatus status = probe.ping(roleInstance);
     assertFalse("Expected a failure but got successful result: " + status,
       status.isSuccess());
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/14b8371b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java
index 9ca3242..d7a9cfd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestServiceApiUtil.java
@@ -34,6 +34,9 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
 
 import static org.apache.slider.util.RestApiConstants.DEFAULT_COMPONENT_NAME;
 import static org.apache.slider.util.RestApiConstants.DEFAULT_UNLIMITED_LIFETIME;
@@ -390,4 +393,52 @@ public class TestServiceApiUtil {
     // original component replaced by external component
     assertNotNull(app.getComponent("comp1"));
   }
+
+  public static void verifyDependencySorting(List<Component> components,
+      Component... expectedSorting) {
+    Collection<Component> actualSorting = ServiceApiUtil.sortByDependencies(
+        components);
+    assertEquals(expectedSorting.length, actualSorting.size());
+    int i = 0;
+    for (Component component : actualSorting) {
+      assertEquals(expectedSorting[i++], component);
+    }
+  }
+
+  @Test
+  public void testDependencySorting() throws IOException {
+    Component a = new Component().name("a");
+    Component b = new Component().name("b");
+    Component c = new Component().name("c");
+    Component d = new Component().name("d").dependencies(Arrays.asList("c"));
+    Component e = new Component().name("e").dependencies(Arrays.asList("b",
+        "d"));
+
+    verifyDependencySorting(Arrays.asList(a, b, c), a, b, c);
+    verifyDependencySorting(Arrays.asList(c, a, b), c, a, b);
+    verifyDependencySorting(Arrays.asList(a, b, c, d, e), a, b, c, d, e);
+    verifyDependencySorting(Arrays.asList(e, d, c, b, a), c, b, a, d, e);
+
+    c.setDependencies(Arrays.asList("e"));
+    try {
+      verifyDependencySorting(Arrays.asList(a, b, c, d, e));
+      Assert.fail(EXCEPTION_PREFIX + "components with dependency cycle");
+    } catch (IllegalArgumentException ex) {
+      assertEquals(String.format(
+          RestApiErrorMessages.ERROR_DEPENDENCY_CYCLE, Arrays.asList(c, d,
+              e)), ex.getMessage());
+    }
+
+    SliderFileSystem sfs = initMock(null);
+    Application application = createValidApplication(null);
+    application.setComponents(Arrays.asList(c, d, e));
+    try {
+      ServiceApiUtil.validateAndResolveApplication(application, sfs);
+      Assert.fail(EXCEPTION_PREFIX + "components with bad dependencies");
+    } catch (IllegalArgumentException ex) {
+      assertEquals(String.format(
+          RestApiErrorMessages.ERROR_DEPENDENCY_INVALID, "b", "e"), ex
+          .getMessage());
+    }
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message