slider-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ste...@apache.org
Subject [16/25] incubator-slider git commit: SLIDER-799 tests are all working
Date Wed, 25 Mar 2015 20:28:45 GMT
SLIDER-799 tests are all working


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ad41b244
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ad41b244
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ad41b244

Branch: refs/heads/develop
Commit: ad41b2444c68a27ab9a5d1a10470da69f74f7bb6
Parents: 3dd2f72
Author: Steve Loughran <stevel@apache.org>
Authored: Thu Mar 19 17:07:50 2015 +0000
Committer: Steve Loughran <stevel@apache.org>
Committed: Thu Mar 19 17:07:50 2015 +0000

----------------------------------------------------------------------
 .../slider/providers/PlacementPolicy.java       |  10 +-
 .../slider/server/appmaster/state/AppState.java |   4 +-
 .../appmaster/state/OutstandingRequest.java     | 120 +++++++++++++------
 .../state/OutstandingRequestTracker.java        |   2 +-
 .../server/appmaster/state/RoleHistory.java     |   3 +-
 .../server/appmaster/state/RoleStatus.java      |  22 ++--
 .../TestMockAppStateDynamicRoles.groovy         |   3 +-
 .../TestMockAppStateRolePlacement.groovy        |   2 +-
 ...tRoleHistoryOutstandingRequestTracker.groovy | 109 ++++++++++++-----
 .../TestRoleHistoryRequestTracking.groovy       |  28 ++---
 .../model/mock/BaseMockAppStateTest.groovy      |   2 +-
 .../appmaster/model/mock/MockFactory.groovy     |  13 +-
 12 files changed, 211 insertions(+), 107 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad41b244/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java b/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java
index dc6c910..4e85a93 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java
@@ -19,12 +19,13 @@
 package org.apache.slider.providers;
 
 /**
- * Placement values
+ * Placement values.
+ * This is nominally a bitmask, though not all values make sense
  */
 public class PlacementPolicy {
 
   /**
-   * Default values
+   * Default value: history used, anti-affinity hinted at on rebuild/flex up
    */
   public static final int DEFAULT = 0;
 
@@ -35,11 +36,12 @@ public class PlacementPolicy {
   public static final int STRICT = 1;
 
   /**
-   * No data locality; do not bother trying to ask for any location
+   * No data locality; do not use placement history
    */
   public static final int NO_DATA_LOCALITY = 2;
+
   /**
-   * Anti-affinity is mandatory. 
+   * Anti-affinity is mandatory. This is not supported in YARN
    */
   public static final int ANTI_AFFINITY_REQUIRED = 4;
   

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad41b244/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index b7ca526..0f07ee9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -766,7 +766,7 @@ public class AppState {
 
     // Add all the existing roles
     for (RoleStatus roleStatus : getRoleStatusMap().values()) {
-      if (roleStatus.getExcludeFromFlexing()) {
+      if (roleStatus.isExcludeFromFlexing()) {
         // skip inflexible roles, e.g AM itself
         continue;
       }
@@ -1792,7 +1792,7 @@ public class AppState {
     log.debug("in reviewRequestAndReleaseNodes()");
     List<AbstractRMOperation> allOperations = new ArrayList<AbstractRMOperation>();
     for (RoleStatus roleStatus : getRoleStatusMap().values()) {
-      if (!roleStatus.getExcludeFromFlexing()) {
+      if (!roleStatus.isExcludeFromFlexing()) {
         List<AbstractRMOperation> operations = reviewOneRole(roleStatus);
         allOperations.addAll(operations);
       }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad41b244/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
index 4fd2933..8c320f0 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
@@ -64,27 +64,32 @@ public final class OutstandingRequest {
    * <p>
    * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long, String)}
    */
-  public AMRMClient.ContainerRequest issuedRequest;
+  private AMRMClient.ContainerRequest issuedRequest;
   
   /**
    * Requested time in millis.
    * <p>
    * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long, String)}
    */
-  public long requestedTimeMillis;
+  private long requestedTimeMillis;
 
   /**
    * Time in millis after which escalation should be triggered..
    * <p>
    * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long, String)}
    */
-  public long escalationTimeoutMillis;
+  private long escalationTimeoutMillis;
 
   /**
    * Has the placement request been escalated?
    */
-  public boolean escalated;
-  
+  private boolean escalated;
+
+  /**
+   * Flag to indicate that escalation is allowed
+   */
+  private boolean mayEscalate;
+
   /**
    * Create a request
    * @param roleId role
@@ -98,7 +103,7 @@ public final class OutstandingRequest {
   }
 
   /**
-   * Create an outsanding request with the given role and hostname
+   * Create an outstanding request with the given role and hostname
    * Important: this is useful only for map lookups -the other constructor
    * with the NodeInstance parameter is needed to generate node-specific
    * container requests
@@ -113,14 +118,40 @@ public final class OutstandingRequest {
 
   /**
    * Is the request located in the cluster, that is: does it have a node.
-   * @return
+   * @return true if a node instance was supplied in the constructor
    */
   public boolean isLocated() {
     return node != null;
   }
-  
+
+  public long getRequestedTimeMillis() {
+    return requestedTimeMillis;
+  }
+
+  public long getEscalationTimeoutMillis() {
+    return escalationTimeoutMillis;
+  }
+
+  public boolean isEscalated() {
+    return escalated;
+  }
+
+  public boolean mayEscalate() {
+    return mayEscalate;
+  }
+
+  public AMRMClient.ContainerRequest getIssuedRequest() {
+    return issuedRequest;
+  }
+
   /**
    * Build a container request.
+   * <p>
+   *  The value of {@link #node} is used to direct a lot of policy. If null,
+   *  placement is relaxed.
+   *  If not null, the choice of whether to use the suggested node
+   *  is based on the placement policy and failure history.
+   * <p>
    * If the request has an address, it is set in the container request
    * (with a flag to enable relaxed priorities).
    * <p>
@@ -134,37 +165,54 @@ public final class OutstandingRequest {
    */
   public synchronized AMRMClient.ContainerRequest buildContainerRequest(
       Resource resource, RoleStatus role, long time, String labelExpression) {
-    String[] hosts;
-    boolean relaxLocality;
+    Preconditions.checkArgument(resource != null, "null `resource` arg");
+    Preconditions.checkArgument(role != null, "null `role` arg");
+
     requestedTimeMillis = time;
     escalationTimeoutMillis = time + role.getPlacementTimeoutSeconds() * 1000;
-    boolean usePlacementHistory = role.isStrictOrAntiAffinePlacement();
-    if (!usePlacementHistory) {
-      // If strict placement does not mandate using placement then check
-      // that the recent failures on this node is not higher than threshold
-      if (node != null) {
-        int numFailuresOnLastHost = node.get(role.getKey()).getFailedRecently();
-        usePlacementHistory = numFailuresOnLastHost <= role.getNodeFailureThreshold();
-        if(!usePlacementHistory) {
-          log.info("Recent node failures {} is higher than threshold {}. Dropping host {}
from preference.",
-                   numFailuresOnLastHost, role.getNodeFailureThreshold(), node.hostname);
-        }
+    String[] hosts;
+    boolean relaxLocality;
+    boolean strictPlacement = role.isStrictPlacement();
+    NodeInstance target = this.node;
+    if (target != null) {
+      // there is a host specified; get its details
+
+      // tell the node it is in play
+      NodeEntry entry = target.getOrCreate(roleId);
+      // failure count
+      int numFailuresOnLastHost = entry != null ? entry.getFailedRecently() : 0;
+
+      // which on non-strict placement may have some effect
+      if (!strictPlacement && numFailuresOnLastHost > role.getNodeFailureThreshold())
{
+        // too many failures for this node
+        log.info("Recent node failures {} is higher than threshold {}. Not requesting host
{}",
+            numFailuresOnLastHost, role.getNodeFailureThreshold(), target.hostname);
+        // reset the target node so this request is downgraded
+        target = null;
       }
     }
 
-    if (node != null && usePlacementHistory) {
+    if (target != null) {
+      // placed request. Hostname is used in request
       hosts = new String[1];
-      hosts[0] = node.hostname;
-      relaxLocality = !role.isStrictOrAntiAffinePlacement();
-      // tell the node it is in play
-      node.getOrCreate(roleId);
+      hosts[0] = target.hostname;
+      // and locality flag is set to false; Slider will decide when
+      // to relax things
+      relaxLocality = false;
+
       log.info("Submitting request for container on {}", hosts[0]);
+      // enable escalation for all but strict placements.
+      mayEscalate = !strictPlacement;
       escalated = false;
     } else {
-      // the placement is implicitly escalated.
-      escalated = true;
+      // no hosts
       hosts = null;
+      // relax locality is mandatory on an unconstrained placement
       relaxLocality = true;
+      // declare that the the placement is implicitly escalated.
+      escalated = true;
+      // and forbid it happening
+      mayEscalate = false;
     }
     Priority pri = ContainerPriority.createPriority(roleId, !relaxLocality);
     issuedRequest = new AMRMClient.ContainerRequest(resource,
@@ -197,7 +245,6 @@ public final class OutstandingRequest {
       nodes = null;
     }
 
-
     AMRMClient.ContainerRequest newRequest =
         new AMRMClient.ContainerRequest(issuedRequest.getCapability(),
             nodes,
@@ -218,14 +265,17 @@ public final class OutstandingRequest {
   }
 
   /**
-   * Query to see if the request is ready to be escalated
+   * Query to see if the request is available and ready to be escalated
    * @param time time to check against
    * @return true if escalation should begin
    */
   public synchronized boolean shouldEscalate(long time) {
-    return issuedRequest != null && !escalated && escalationTimeoutMillis
< time;
+    return mayEscalate
+           && !escalated
+           && issuedRequest != null
+           && escalationTimeoutMillis < time;
   }
-  
+
   /**
    * Equality is on hostname and role
    * @param o other
@@ -265,14 +315,16 @@ public final class OutstandingRequest {
   }
 
   @Override
-  public synchronized String toString() {
+  public String toString() {
     final StringBuilder sb = new StringBuilder("OutstandingRequest{");
     sb.append("roleId=").append(roleId);
     sb.append(", node=").append(node);
     sb.append(", hostname='").append(hostname).append('\'');
+    sb.append(", issuedRequest=").append(issuedRequest);
     sb.append(", requestedTimeMillis=").append(requestedTimeMillis);
-    sb.append(", escalationTimeoutMillis=").append(escalationTimeoutMillis);
+    sb.append(", mayEscalate=").append(mayEscalate);
     sb.append(", escalated=").append(escalated);
+    sb.append(", escalationTimeoutMillis=").append(escalationTimeoutMillis);
     sb.append('}');
     return sb.toString();
   }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad41b244/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
index 48f6e57..959cb1f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
@@ -251,7 +251,7 @@ public class OutstandingRequestTracker {
         if (outstandingRequest.shouldEscalate(now)) {
 
           // time to escalate
-          CancelSingleRequest cancel = new CancelSingleRequest(outstandingRequest.issuedRequest);
+          CancelSingleRequest cancel = new CancelSingleRequest(outstandingRequest.getIssuedRequest());
           operations.add(cancel);
           AMRMClient.ContainerRequest escalated =
               outstandingRequest.escalate();

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad41b244/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 6aefc47..a1b54c7 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -481,7 +481,8 @@ public class RoleHistory {
    */
   @VisibleForTesting
   public synchronized NodeInstance findNodeForNewInstance(RoleStatus role) {
-    if (role.getNoDataLocality()) {
+    if (!role.isPlacementDesired()) {
+      // no data locality policy
       return null;
     }
     int roleKey = role.getKey();

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad41b244/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
index 4ee8fad..899948f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -87,24 +87,24 @@ public final class RoleStatus implements Cloneable {
     return providerRole.nodeFailureThreshold;
   }
 
-  public boolean getExcludeFromFlexing() {
-    return 0 != (getPlacementPolicy() & PlacementPolicy.EXCLUDE_FROM_FLEXING);
+  public boolean isExcludeFromFlexing() {
+    return hasPlacementPolicy(PlacementPolicy.EXCLUDE_FROM_FLEXING);
   }
-  
-  public boolean getNoDataLocality() {
-    return 0 != (getPlacementPolicy() & PlacementPolicy.NO_DATA_LOCALITY);
-  }
-  
+
   public boolean isStrictPlacement() {
-    return 0 != (getPlacementPolicy() & PlacementPolicy.STRICT);
+    return hasPlacementPolicy(PlacementPolicy.STRICT);
   }
 
   public boolean isAntiAffinePlacement() {
-    return 0 != (getPlacementPolicy() & PlacementPolicy.ANTI_AFFINITY_REQUIRED);
+    return hasPlacementPolicy(PlacementPolicy.ANTI_AFFINITY_REQUIRED);
+  }
+
+  public boolean hasPlacementPolicy(int policy) {
+    return 0 != (getPlacementPolicy() & policy);
   }
 
-  public boolean isStrictOrAntiAffinePlacement() {
-    return isStrictPlacement() || isAntiAffinePlacement();
+  public boolean isPlacementDesired() {
+    return !hasPlacementPolicy(PlacementPolicy.NO_DATA_LOCALITY);
   }
 
   public synchronized int getDesired() {

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad41b244/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
index 13ecf13..ee4abd6 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
@@ -168,7 +168,7 @@ class TestMockAppStateDynamicRoles extends BaseMockAppStateTest
     assert instances.size() == 1
 
     def instanceA = instances.find { RoleInstance instance ->
-      instance.roleId = ID4
+      instance.roleId == ID4
     }
     assert instanceA
     def hostname = RoleHistoryUtils.hostnameOf(instanceA.container)
@@ -193,7 +193,6 @@ class TestMockAppStateDynamicRoles extends BaseMockAppStateTest
     def actions = appState.reviewRequestAndReleaseNodes()
     assert actions.size() == 1
 
-    assertRelaxLocalityFlag(ID4, "", true, actions)
     ContainerRequestOperation cro = (ContainerRequestOperation) actions[0]
     def nodes = cro.request.nodes
     assert nodes.size() == 1

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad41b244/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
index e9de390..8fd9858 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
@@ -101,7 +101,7 @@ class TestMockAppStateRolePlacement extends BaseMockAppStateTest
     AMRMClient.ContainerRequest request2 = operation.request
     assert request2 != null
     assert request2.nodes[0] == containerHostname
-    assert request2.relaxLocality
+    assert !request2.relaxLocality
     engine.execute(ops)
 
   }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad41b244/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
index 2fe6763..97d970d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
@@ -19,6 +19,7 @@
 package org.apache.slider.server.appmaster.model.history
 
 import org.apache.hadoop.yarn.api.records.Resource
+import org.apache.hadoop.yarn.client.api.AMRMClient
 import org.apache.slider.providers.PlacementPolicy
 import org.apache.slider.providers.ProviderRole
 import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
@@ -26,6 +27,7 @@ import org.apache.slider.server.appmaster.model.mock.MockResource
 import org.apache.slider.server.appmaster.operations.AbstractRMOperation
 import org.apache.slider.server.appmaster.operations.CancelSingleRequest
 import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
+import org.apache.slider.server.appmaster.state.NodeEntry
 import org.apache.slider.server.appmaster.state.NodeInstance
 import org.apache.slider.server.appmaster.state.OutstandingRequest
 import org.apache.slider.server.appmaster.state.OutstandingRequestTracker
@@ -64,9 +66,9 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest
{
   
   @Test
   public void testResetEntries() throws Throwable {
-    OutstandingRequest r1 = tracker.newRequest(host1, 0)
-    OutstandingRequest r2 = tracker.newRequest(host2, 0)
-    OutstandingRequest r3 = tracker.newRequest(host1, 1)
+    tracker.newRequest(host1, 0)
+    tracker.newRequest(host2, 0)
+    tracker.newRequest(host1, 1)
     List<NodeInstance> canceled = tracker.resetOutstandingRequests(0)
     assert canceled.size() == 2
     assert canceled.contains(host1)
@@ -78,26 +80,17 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest
{
     assert tracker.resetOutstandingRequests(1).size() == 1
   }
 
-/*
-  @Override
-  AggregateConf buildInstanceDefinition() {
-    def aggregateConf = super.buildInstanceDefinition()
-    def component0 = aggregateConf.resourceOperations.getMandatoryComponent(ROLE0)
-    component0.set(ResourceKeys.COMPONENT_PLACEMENT_POLICY, PlacementPolicy.STRICT)
-    return aggregateConf
-  }
-*/
 
   @Test
   public void testEscalation() throws Throwable {
-    ProviderRole providerRole1 = role1Status.providerRole
-    assert providerRole1.placementPolicy == PlacementPolicy.STRICT;
-    // first request
-    final def (res1, outstanding1) = newRequest(role1Status)
-    final def initialRequest = outstanding1.buildContainerRequest(res1, role1Status, 0, null)
-    assert outstanding1.issuedRequest != null;
-    assert outstanding1.located
-    assert !outstanding1.escalated
+
+    // first request: default placement
+    assert role0Status.placementPolicy == PlacementPolicy.DEFAULT;
+    final def (res0, outstanding0) = newRequest(role0Status)
+    final def initialRequest = outstanding0.buildContainerRequest(res0, role0Status, 0, null)
+    assert outstanding0.issuedRequest != null;
+    assert outstanding0.located
+    assert !outstanding0.escalated
     assert !initialRequest.relaxLocality
     assert tracker.listOutstandingRequests().size() == 1
 
@@ -109,32 +102,86 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest
{
     def (res2, outstanding2) = newRequest(role2Status)
 
     // simulate some time escalation of role 1 MUST now be triggered
-    List<AbstractRMOperation> escalations =
-        tracker.escalateOutstandingRequests(providerRole1.placementTimeoutSeconds * 1000
+ 500 )
+    final def interval = role0Status.placementTimeoutSeconds * 1000 + 500
+    def now = interval
+    final List<AbstractRMOperation> escalations = tracker.escalateOutstandingRequests(now)
 
-    assert outstanding1.escalated
+    assert outstanding0.escalated
     assert !outstanding2.escalated
 
     // two entries
     assert escalations.size() == 2;
     final def e1 = escalations[0]
     assert  e1 instanceof CancelSingleRequest
-    CancelSingleRequest cancel = (CancelSingleRequest) e1
+    final CancelSingleRequest cancel = (CancelSingleRequest) e1
     assert initialRequest == cancel.request
     final def e2 = escalations[1]
     assert e2 instanceof ContainerRequestOperation;
     final def escRequest = (ContainerRequestOperation) e2
-    def req2 = escRequest.request
-    assert req2.relaxLocality
+    assert escRequest.request.relaxLocality
+
+    // build that second request from an anti-affine entry
+    // these get placed as well
+    now += interval
+    final def containerReq2 = outstanding2.buildContainerRequest(res2, role2Status, now,
null)
+    // escalate a little bit more
+    final List<AbstractRMOperation> escalations2 = tracker.escalateOutstandingRequests(now)
+    // and expect no new entries
+    assert escalations2.size() == 0
 
-    def (res3, outstanding3) = newRequest(role2Status)
-    outstanding3.buildContainerRequest(res3, role2Status, 0, null)
+    // go past the role2 timeout
+    now += role2Status.placementTimeoutSeconds * 1000 + 500
+    // escalate a little bit more
+    final List<AbstractRMOperation> escalations3 = tracker.escalateOutstandingRequests(now)
+    // and expect another escalation
+    assert escalations3.size() == 2
+    assert outstanding2.escalated
+
+    // finally add a strict entry to th emix
+    def (res3, outstanding3) = newRequest(role1Status)
+    final ProviderRole providerRole1 = role1Status.providerRole
+    assert providerRole1.placementPolicy == PlacementPolicy.STRICT
+    now += interval
+    assert !outstanding3.mayEscalate
+    final List<AbstractRMOperation> escalations4 = tracker.escalateOutstandingRequests(now)
+    assert escalations4.empty
 
-    List<AbstractRMOperation> escalations2 =
-        tracker.escalateOutstandingRequests(providerRole1.placementTimeoutSeconds * 1000
+ 500)
-    assert escalations2.size() == 0
   }
 
+  @Test
+  public void testPlacementSkipsFailures() throws Throwable {
+    final def (res0, outstanding0) = newRequest(role0Status)
+    def entry = host1.getOrCreate(role0Status.key)
+    entry.containerCompleted(false)
+    entry.containerCompleted(false)
+    entry.containerCompleted(false)
+    assert entry.failedRecently == 3
+    final AMRMClient.ContainerRequest initialRequest = outstanding0.buildContainerRequest(res0,
role0Status, 0, null)
+    assert initialRequest.relaxLocality
+    assert initialRequest.nodes == null
+  }
+
+  @Test
+  public void testStrictPlacementDoesntSkipFailures() throws Throwable {
+    def roleStatus = role1Status
+    assert roleStatus.strictPlacement
+    final def (res0, outstanding0) = newRequest(roleStatus)
+    def entry = host1.getOrCreate(roleStatus.key)
+    entry.containerCompleted(false)
+    entry.containerCompleted(false)
+    entry.containerCompleted(false)
+    assert entry.failedRecently == 3
+    final AMRMClient.ContainerRequest initialRequest = outstanding0.buildContainerRequest(res0,
+        roleStatus, 0, null)
+    assert !initialRequest.relaxLocality
+    assert initialRequest.nodes[0] == host1.hostname
+  }
+
+  /**
+   * Create a new request (always against host1)
+   * @param r
+   * @return
+   */
   public def newRequest(RoleStatus r) {
     final Resource res2 = new MockResource()
     appState.buildResourceRequirements(r, res2)

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad41b244/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
index e593eab..e84dfce 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
@@ -122,7 +122,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
   @Test
   public void testRequestedNodeIntoReqList() throws Throwable {
     AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
-    List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests
+    List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests()
     assert requests.size() == 1
     assert age3Active0.hostname == requests[0].hostname
   }
@@ -130,28 +130,28 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
   @Test
   public void testCompletedRequestDropsNode() throws Throwable {
     AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
-    List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests
+    List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests()
     assert requests.size() == 1
     String hostname = requests[0].hostname
     assert age3Active0.hostname == hostname
     assert hostname == req.nodes[0]
     MockContainer container = factory.newContainer(req, hostname)
     assert roleHistory.onContainerAllocated(container , 2, 1)
-    assert roleHistory.listOutstandingPlacedRequests.empty
+    assert roleHistory.listOutstandingPlacedRequests().empty
   }
   
   @Test
   public void testTwoRequests() throws Throwable {
     AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
     AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
-    List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests
+    List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests()
     assert requests.size() == 2
     MockContainer container = factory.newContainer(req, req.nodes[0])
     assert roleHistory.onContainerAllocated(container , 2, 1)
-    assert roleHistory.listOutstandingPlacedRequests.size() == 1
+    assert roleHistory.listOutstandingPlacedRequests().size() == 1
     container = factory.newContainer(req2, req2.nodes[0])
     assert roleHistory.onContainerAllocated(container, 2, 2)
-    assert roleHistory.listOutstandingPlacedRequests.empty
+    assert roleHistory.listOutstandingPlacedRequests().empty
   }
 
     
@@ -160,22 +160,22 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
     AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
     AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
     AMRMClient.ContainerRequest req3 = roleHistory.requestNode(roleStatus, resource)
-    List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests
+    List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests()
     assert requests.size() == 2
     MockContainer container = factory.newContainer(req, req.nodes[0])
     assert roleHistory.onContainerAllocated(container , 2, 1)
-    assert roleHistory.listOutstandingPlacedRequests.size() == 1
+    assert roleHistory.listOutstandingPlacedRequests().size() == 1
     
     container = factory.newContainer(req3, "three")
     assert !roleHistory.onContainerAllocated(container, 3, 2)
-    assert roleHistory.listOutstandingPlacedRequests.size() == 1
+    assert roleHistory.listOutstandingPlacedRequests().size() == 1
     
     // the final allocation will trigger a cleanup
     container = factory.newContainer(req2, "four")
     // no node dropped
     assert !roleHistory.onContainerAllocated(container, 3, 3)
     // yet the list is now empty
-    assert roleHistory.listOutstandingPlacedRequests.empty
+    assert roleHistory.listOutstandingPlacedRequests().empty
 
     // and the remainder goes onto the available list
     List<NodeInstance> a2 = roleHistory.cloneAvailableList(0)
@@ -189,17 +189,17 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
     AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
     AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
     AMRMClient.ContainerRequest req3 = roleHistory.requestNode(roleStatus, resource)
-    assert roleHistory.listOutstandingPlacedRequests.size() == 2
+    assert roleHistory.listOutstandingPlacedRequests().size() == 2
     assert req3.nodes == null
     MockContainer container = factory.newContainer(req, req.nodes[0])
     assert roleHistory.onContainerAllocated(container , 3, 1)
-    assert roleHistory.listOutstandingPlacedRequests.size() == 1
+    assert roleHistory.listOutstandingPlacedRequests().size() == 1
     container = factory.newContainer(req2, req2.nodes[0])
     assert roleHistory.onContainerAllocated(container, 3, 2)
-    assert roleHistory.listOutstandingPlacedRequests.empty
+    assert roleHistory.listOutstandingPlacedRequests().empty
     container = factory.newContainer(req3, "three")
     assert !roleHistory.onContainerAllocated(container, 3, 3)
-    assert roleHistory.listOutstandingPlacedRequests.empty
+    assert roleHistory.listOutstandingPlacedRequests().empty
   }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad41b244/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
index 6c83c55..f30fce6 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
@@ -135,7 +135,7 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements
MockRoles
   /**
    * Build a role instance from a container assignment
    * @param assigned
-   * @return
+   * @return the instance
    */
   RoleInstance roleInstance(ContainerAssignment assigned) {
     Container target = assigned.container

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad41b244/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
index 37f4021..06bc10c 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
@@ -40,18 +40,23 @@ class MockFactory implements MockRoles {
 
   public static final ProviderRole PROVIDER_ROLE0 = new ProviderRole(
       MockRoles.ROLE0,
-      0)
+      0,
+      PlacementPolicy.DEFAULT,
+      2,
+      1)
+  // role 1 is strict. timeout should be irrelevant; same as failures
   public static final ProviderRole PROVIDER_ROLE1 = new ProviderRole(
       MockRoles.ROLE1,
       1,
       PlacementPolicy.STRICT,
-      3,
+      2,
       1)
+  // role 2: longer delay
   public static final ProviderRole PROVIDER_ROLE2 = new ProviderRole(
       MockRoles.ROLE2,
       2,
       PlacementPolicy.ANTI_AFFINITY_REQUIRED,
-      4,
+      2,
       2)
   int appIdCount;
   int attemptIdCount;
@@ -180,8 +185,6 @@ class MockFactory implements MockRoles {
   def roleMap(int count) {
     return [
         (ResourceKeys.COMPONENT_INSTANCES):count.toString(),
-        (ResourceKeys.COMPONENT_PLACEMENT_POLICY):"${PlacementPolicy.STRICT}".toString()
-
     ]
   }
 


Mime
View raw message